Upload 38 files

.gitignore

@@ -1,22 +1,22 @@
-# Ignore .env file
-.env
-__pycache__/
-newenv
-backupgoal.py
-backupgoal2.py
-backupresearch.py
-goals.py
-goals3.py
-research_assistant_dashboard2.py
-tempCodeRunnerFile.py
-all_chat_histories.json
-all_chat_histories2.json
-analytics.ipynb
-chat_history.csv
-harshal.py
-course_creation.py
-topics.json
-new_analytics.json
-new_analytics2.json
-pre_class_analytics.py
+# Ignore .env file
+.env
+__pycache__/
+newenv
+backupgoal.py
+backupgoal2.py
+backupresearch.py
+goals.py
+goals3.py
+research_assistant_dashboard2.py
+tempCodeRunnerFile.py
+all_chat_histories.json
+all_chat_histories2.json
+analytics.ipynb
+chat_history.csv
+harshal.py
+course_creation.py
+topics.json
+new_analytics.json
+new_analytics2.json
+pre_class_analytics.py
 sample_files/

Columns.py

@@ -0,0 +1,247 @@
+
+import streamlit as st
+import pandas as pd
+import PyPDF2
+import io
+import os
+from dotenv import load_dotenv
+import requests
+import time
+from mistralai import Mistral
+from typing import List, Dict
+from fpdf import FPDF
+
+load_dotenv()
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+MISTRAL_API_URL = "https://api.mistral.ai/v1/completions"
+
+# Initialize the Mistral client
+client = Mistral(api_key=MISTRAL_API_KEY)
+
+def call_mistral_api(prompt: str) -> str:
+    """Call Mistral AI with a prompt, return the text response if successful."""
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt}
+    ]
+    tools = []  # Add any tools if necessary
+
+    try:
+        # Make the API call
+        response = client.chat.complete(
+            model="mistral-large-latest",
+            messages=messages,
+            tools=tools,
+            tool_choice="any",
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        print(f"API Error: {str(e)}")
+        return ""
+
+def process_dataframe(df: pd.DataFrame) -> pd.DataFrame:
+    """Process the DataFrame and return a DataFrame with analysis results."""
+    print("Processing DataFrame...")
+    # Initialize results dictionary
+    results = []
+
+    # Process each column starting from the third column
+    for i, column in enumerate(df.columns[2:], start=2):
+        print(f"Processing column: {column}")
+        # Extract text from column and attach values from the first and second columns
+        text = " ".join(
+            f"Column1-{row[df.columns[0]]}, Column2-{row[df.columns[1]]}, {value}"
+            for _, row in df.iterrows()
+            for value in [row[column]]
+            if pd.notna(value)
+        )
+
+        # Generate prompt
+        prompt = f"You are a Professional Researcher and Analyser with 10 yrs of Experience. Find details and Elaborate on Top Trends,Patterns ,Highlight Theories and Method in this topic.Support your answer with rightful evidence of corresponding DOI/SrNo and Frequency(how many times same topic repeated and in which papers):Make sure to limit the answer within 400 words ({column}):\n\n{text}"
+        
+        # Call Mistral API
+        result1 = call_mistral_api(prompt)
+        prompt1=f"""This result was the reponse of an earlier prompt Result -{result1}, Fact check the result with my original data -({column}):\n\n{text}. Return the refined Result(after careful fact checking and finding adequate evidence within the original data) , Make sure the meaning/structure of the Result doesnt change,only false/low evidence statements get eliminated.Limit the response to 400 words.MAKE SURE THERE IS NO CONTEXT CHANGE AND MEANING REMAINS SAME JUST WITH GOOD EVIDENCE AND REFINED RESULT. """
+        result=call_mistral_api(prompt1)
+        results.append({"Column": column, "Result": result})
+
+    # Create DataFrame from results
+    results_df = pd.DataFrame(results)
+    print("DataFrame processing complete.")
+    return results_df
+
+def split_dataframe(df: pd.DataFrame, max_rows: int = 52) -> List[pd.DataFrame]:
+    """
+    Split a DataFrame into multiple smaller DataFrames, each having a maximum of `max_rows` rows.
+    
+    Args:
+        df (pd.DataFrame): The original DataFrame to be split.
+        max_rows (int): The maximum number of rows for each smaller DataFrame (excluding the header row).
+    
+    Returns:
+        List[pd.DataFrame]: A list of smaller DataFrames.
+    """
+    print("Splitting DataFrame...")
+    # Calculate the number of splits needed
+    num_splits = (len(df) + max_rows - 1) // max_rows
+    
+    # Split the DataFrame
+    split_dfs = [df.iloc[i * max_rows:(i + 1) * max_rows].reset_index(drop=True) for i in range(num_splits)]
+    print(f"DataFrame split into {len(split_dfs)} parts.")
+    return split_dfs
+
+def generate_professional_review(df1: pd.DataFrame) -> str:
+    """
+    Generate a professional literature review, trends analysis, TSM/ADO analysis, gaps, theories, and frameworks
+    based on DOI and Serial Number as key value pairs.
+    
+    Args:
+        df1 (pd.DataFrame): The first DataFrame.
+        df2 (pd.DataFrame): The second DataFrame.
+    
+    Returns:
+        str: The generated analysis text.
+    """
+    print("Generating professional review...")
+    # Concatenate DataFrames
+    
+
+    # Convert the concatenated DataFrame to a string format suitable for the prompt
+    context = df1.to_string(index=True)
+    
+    # Generate a single prompt for the analysis
+    prompt = f"""Generate a professional literature review, trends analysis, TCM ADO (Theories,Context,Method ,Ancedents,Decisions,Outcomes), gaps, theories, and frameworks
+    based on the following data , If you find evidence as proper DOI make sure you analyze the whole
+    table with more DOI,Serial No and find more evidence.Always give supporting evidence for your literature review,TCM ADO analysis,trends ,frameworks,
+    check DOIs and find more evidence as inference again.Make sure the review is as professional as possible.Limit the answer to 500 words and only highlight the most imp trends with supporting evidence of DOI/SrNo and frequency(how many papers used that and top 2 DOI of that),Limit it to 500 words.Make sure all important details/frequently repeating trends/methods are highlighted.:\n\n{context}."""
+  
+
+    # Call Mistral API
+    result = call_mistral_api(prompt)
+    print("Professional review generated.")
+    return result
+
+
+def main():
+    st.title("Research Corpus Synthesis Tool")
+
+    # Logout button
+    if st.button("Logout", use_container_width=True):
+        for key in st.session_state.keys():
+            del st.session_state[key]
+        st.rerun()
+
+    # File uploader
+    uploaded_file = st.file_uploader("Upload CSV file", type="csv")
+
+    if uploaded_file:
+        if st.button("Process CSV"):
+            print("CSV file uploaded.")
+            # Initialize progress bar
+            progress_bar = st.progress(0)
+            status_text = st.empty()
+
+            # Read CSV file into DataFrame
+            df = pd.read_csv(uploaded_file)
+            print("CSV file read into DataFrame.")
+
+            # Split DataFrame into smaller DataFrames
+            split_dfs = split_dataframe(df, max_rows=52)
+
+            # Initialize variable to concatenate all generated reviews
+            concatenated_reviews = ""
+
+            # Process each smaller DataFrame
+            for i, split_df in enumerate(split_dfs):
+                status_text.text(f"Processing part {i + 1} of {len(split_dfs)}")
+                print(f"Processing part {i + 1} of {len(split_dfs)}")
+
+                # Process the smaller DataFrame
+                processed_df = process_dataframe(split_df)
+
+                # Generate professional review
+                review = generate_professional_review(processed_df)
+
+                # Concatenate the generated review
+                concatenated_reviews += review + "\n\n"
+
+                # Update progress
+                progress = (i + 1) / len(split_dfs)
+                progress_bar.progress(progress)
+                st.write(i)
+                st.write(review)
+                
+               
+                
+               
+
+            # Generate final analysis based on the concatenated reviews
+            final_prompt = f"""
+            Given is a consolidated research review of a huge number of research papers (evidence is DOI, Serial No). Perform this:
+            Given as a context is a table of analyzing trends/frameworks analysis of a huge corpus of papers specific to the columns.
+            Analyze the table properly and create a professional and accurate literature review (Ensure to cite DOI as evidence).
+
+            Subheadings for Literature Review :
+            1. Introduction
+               ○ Overview of the main topic or concept.
+               ○ Key research questions or objectives.
+            2. Theoretical Foundations
+               ○ Exploration of dominant theories related to the topic.
+               ○ Domain-specific theoretical applications.
+            3. Contextual Analysis
+               ○ Geographic contexts and challenges.
+               ○ Sectoral applications and digital infrastructure readiness.
+            4. Methodological Approaches
+               ○ Qualitative, quantitative, and mixed-methods approaches used in research.
+            5. Discussion and Future Research
+               ○ Current challenges and limitations.
+               ○ Potential areas for future study.
+            6. Conclusion
+               ○ Summary of findings.
+               ○ Implications and future directions.
+
+            TCM-ADO Framework in Research Analysis and Literature Review:
+            Theory
+               Theoretical foundations driving the research.
+               ● Focus on identifying and analyzing the conceptual models or frameworks that underpin the study.
+               ● Establish the intellectual basis and rationale for the research direction.
+            Context
+               Situational and environmental factors shaping the research.
+               ● Emphasis on geographic, sectoral, cultural, and infrastructural dimensions influencing the implementation or findings.
+               ● Examples include urban versus rural settings, digital infrastructure readiness, or policy landscapes.
+               ● Objective: To understand how external conditions impact the dynamics and applicability of the research.
+            Method
+               Research methodologies and analytical approaches utilized.
+               ● Covers the selection of qualitative, quantitative, or mixed-method approaches, along with tools and techniques employed.
+               ● Objective: To ensure methodological rigor and the validity of findings.
+            Antecedents
+            Pre-existing conditions enabling or constraining research or implementation.
+            ● Includes factors such as technological infrastructure, stakeholder preparedness, and
+            regulatory frameworks.
+            ● To identify critical prerequisites that influence the starting point of the research or
+            initiative.
+            Decisions
+            Strategic choices made throughout the implementation or research process.
+            ● Involves critical decision points in areas like technology adoption, governance
+            frameworks, and operational strategies.
+            ● analyze how informed decision-making shapes the trajectory and success of the project.
+            Outcomes
+            Results and impacts observed as a consequence of the initiative or study.
+            ● Evaluates direct and indirect contributions to the research objectives or broader societal
+            goals.
+            ● assess the effectiveness and long-term implications of the research or project outcomes.
+            """
+
+            final_result = call_mistral_api(final_prompt)
+            print("Final analysis generated.")
+
+            # Display the final result
+            st.subheader("Final Analysis")
+            st.write(final_result)
+
+            status_text.text("Processing complete!")
+            progress_bar.progress(1.0)
+            print("Processing complete.")
+
+if __name__ == "__main__":
+    main()

Research Paper Attributes.txt

@@ -0,0 +1,98 @@
+Review Based Paper
+Title TEXT,
+Publication TEXT,
+Journal_Conference TEXT,
+Abstract TEXT,
+Keywords TEXT,
+Author TEXT
+Date_of_Publication TEXT,
+Intro TEXT,
+Literature_Review TEXT,
+Body: TEXT 
+Protocol: TEXT
+Search String: TEXT 
+Included Studies: TEXT
+Data Collection and Analysis Methods: TEXT 
+Data Extraction Table: TEXT
+Synthesis and Analysis: TEXT 
+Conclusion
+Limitations
+Results
+References
+ 
+Risk of Bias Assessment:Opinion/Perspective Based Paper
+Title TEXT,
+Publication TEXT,
+Journal_Conference TEXT,
+Abstract TEXT,
+Keywords TEXT,
+Author TEXT,
+Date_of_Publication TEXT,
+Intro TEXT,
+Literature_Review TEXT
+Introduction: TEXT 
+Body: TEXT
+Results and Discussion:TEXT
+Conclusion: TEXT 
+References: TEXT 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Empirical Research Paper
+Title TEXT,
+Publication TEXT,
+Journal_Conference TEXT,
+Abstract TEXT,
+Keywords TEXT,
+Author TEXT,
+Date_of_Publication TEXT,
+Intro TEXT,
+Literature_Review TEXT
+Introduction: TEXT 
+Body: TEXT
+ Methodology: TEXT
+        Participants: TEXT - Describes the sample and the sampling methods used.
+Survey Instrument: TEXT - Describes the design and development of the survey questionnaire.
+Data Collection: TEXT - Explains how the survey data was collected.
+Data Analysis: TEXT - Details the statistical techniques used to analyze the data.
+
+
+Results and Discussion:TEXT
+Conclusion: TEXT 
+References: TEXT 
+Research Paper (Other)
+Title TEXT,
+Publication TEXT,
+Journal_Conference TEXT,
+Abstract TEXT,
+Keywords TEXT,
+Author TEXT,
+Date_of_Publication TEXT,
+Intro TEXT,
+Literature_Review TEXT,
+Research_Models_Used TEXT,
+Methodology TEXT,
+Discussion TEXT,
+Future_Scope TEXT,
+Theory TEXT,
+Independent_Variables TEXT,
+nof_Independent_Variables INTEGER,
+Dependent_Variables TEXT,
+nof_Dependent_Variables INTEGER,
+Control_Variables TEXT,
+Extraneous_Variables TEXT,
+nof_Control_Variables INTEGER,
+nof_Extraneous_Variables INTEGER

analytics.py

@@ -1,97 +1,97 @@
-import os
-import pandas as pd
-import numpy as np
-from numpy.linalg import norm
-from pymongo import MongoClient
-import openai
-from openai import OpenAI
-import streamlit as st
-from datetime import datetime
-
-# MongoDB connection
-MONGO_URI = os.getenv('MONGO_URI')
-
-client = MongoClient(MONGO_URI)
-db = client['digital_nova']
-themes_collection = db['themes']
-corpus_collection = db['corpus']
-vectors_collection = db['vectors']  # Reference to 'vectors' collection
-users_collection = db['users']
-
-# Function to create embeddings
-def create_embeddings(text, openai_api_key):
-    client = OpenAI(api_key=openai_api_key)
-    response = client.embeddings.create(
-        input=text,
-        model="text-embedding-3-small"
-    )
-    return response.data[0].embedding
-
-# Function to calculate cosine similarity
-def cosine_similarity(v1, v2):
-    v1 = np.array(v1)
-    v2 = np.array(v2)
-    dot_product = np.dot(v1, v2)
-    norm_product = norm(v1) * norm(v2)
-    return dot_product / norm_product if norm_product != 0 else 0
-
-def derive_analytics(goal, reference_text, openai_api_key, context=None, synoptic=None):
-    """
-    Analyze subjective answers with respect to pre-class materials and synoptic, and provide detailed feedback
-    
-    Args:
-        goal (str): Analysis objective
-        reference_text (str): Student's answer text
-        openai_api_key (str): OpenAI API key
-        context (str, optional): Pre-class material content for comparison
-        synoptic (str, optional): Synoptic content for evaluation
-    """
-    template = f"""Given a student's answer to a subjective question, analyze it following these specific guidelines. Compare it with the provided pre-class materials and synoptic (if available) to assess correctness and completeness.
-
-    1. Analyze the text as an experienced educational assessor, considering:
-       - Conceptual understanding
-       - Factual accuracy
-       - Completeness of response
-       - Use of relevant terminology
-       - Application of concepts
-
-    2. Structure the output in markdown with two sections:
-
-    **Correctness Assessment**
-    - Rate overall correctness on a scale of 1-10
-
-    **Evidence-Based Feedback**
-    - Provide specific evidence from the student's answer to justify the score reduction
-    - Highlight the exact lines or phrases that need improvement
-
-    Pre-class Materials Context:
-    {context if context else "No reference materials provided"}
-
-    Synoptic:
-    {synoptic if synoptic else "No synoptic provided"}
-
-    Student's Answer:
-    {reference_text}
-
-    Rules:
-    - Base assessment strictly on provided content
-    - Be specific in feedback and suggestions
-    """
-    
-    # Initialize OpenAI client
-    client = OpenAI(api_key=openai_api_key)
-
-    try:
-        response = client.chat.completions.create(
-            model="gpt-4-0125-preview",
-            messages=[
-                {"role": "system", "content": "You are an educational assessment expert."},
-                {"role": "user", "content": template}
-            ],
-            temperature=0.7
-        )
-        analysis = response.choices[0].message.content
-        return analysis
-    except Exception as e:
-        print(f"Error in generating analysis with OpenAI: {str(e)}")
-        return "Error generating analysis"
+import os
+import pandas as pd
+import numpy as np
+from numpy.linalg import norm
+from pymongo import MongoClient
+import openai
+from openai import OpenAI
+import streamlit as st
+from datetime import datetime
+
+# MongoDB connection
+MONGO_URI = os.getenv('MONGO_URI')
+
+client = MongoClient(MONGO_URI)
+db = client['digital_nova']
+themes_collection = db['themes']
+corpus_collection = db['corpus']
+vectors_collection = db['vectors']  # Reference to 'vectors' collection
+users_collection = db['users']
+
+# Function to create embeddings
+def create_embeddings(text, openai_api_key):
+    client = OpenAI(api_key=openai_api_key)
+    response = client.embeddings.create(
+        input=text,
+        model="text-embedding-3-small"
+    )
+    return response.data[0].embedding
+
+# Function to calculate cosine similarity
+def cosine_similarity(v1, v2):
+    v1 = np.array(v1)
+    v2 = np.array(v2)
+    dot_product = np.dot(v1, v2)
+    norm_product = norm(v1) * norm(v2)
+    return dot_product / norm_product if norm_product != 0 else 0
+
+def derive_analytics(goal, reference_text, openai_api_key, context=None, synoptic=None):
+    """
+    Analyze subjective answers with respect to pre-class materials and synoptic, and provide detailed feedback
+    
+    Args:
+        goal (str): Analysis objective
+        reference_text (str): Student's answer text
+        openai_api_key (str): OpenAI API key
+        context (str, optional): Pre-class material content for comparison
+        synoptic (str, optional): Synoptic content for evaluation
+    """
+    template = f"""Given a student's answer to a subjective question, analyze it following these specific guidelines. Compare it with the provided pre-class materials and synoptic (if available) to assess correctness and completeness.
+
+    1. Analyze the text as an experienced educational assessor, considering:
+       - Conceptual understanding
+       - Factual accuracy
+       - Completeness of response
+       - Use of relevant terminology
+       - Application of concepts
+
+    2. Structure the output in markdown with two sections:
+
+    **Correctness Assessment**
+    - Rate overall correctness on a scale of 1-10
+
+    **Evidence-Based Feedback**
+    - Provide specific evidence from the student's answer to justify the score reduction
+    - Highlight the exact lines or phrases that need improvement
+
+    Pre-class Materials Context:
+    {context if context else "No reference materials provided"}
+
+    Synoptic:
+    {synoptic if synoptic else "No synoptic provided"}
+
+    Student's Answer:
+    {reference_text}
+
+    Rules:
+    - Base assessment strictly on provided content
+    - Be specific in feedback and suggestions
+    """
+    
+    # Initialize OpenAI client
+    client = OpenAI(api_key=openai_api_key)
+
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4-0125-preview",
+            messages=[
+                {"role": "system", "content": "You are an educational assessment expert."},
+                {"role": "user", "content": template}
+            ],
+            temperature=0.7
+        )
+        analysis = response.choices[0].message.content
+        return analysis
+    except Exception as e:
+        print(f"Error in generating analysis with OpenAI: {str(e)}")
+        return "Error generating analysis"

assignment_evaluation.py

@@ -0,0 +1,261 @@
+# assignment_evaluation.py
+
+import streamlit as st
+from datetime import datetime
+from pymongo import MongoClient
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+from bson import ObjectId
+
+load_dotenv()
+
+# MongoDB setup
+MONGO_URI = os.getenv('MONGO_URI')
+client = MongoClient(MONGO_URI)
+db = client["novascholar_db"]
+assignments_collection = db["assignments"]
+assignment_evaluation_collection = db["assignment_evaluation"]
+resources_collection = db["resources"]
+students_collection = db["students"]
+
+def evaluate_assignment(session_id, student_id, assignment_id):
+    """
+    Generate evaluation and analysis for submitted assignments
+    """
+    try:
+        # Fetch assignment and student submission
+        assignment = assignments_collection.find_one({"_id": assignment_id})
+        if not assignment:
+            return None
+
+        # Find student's submission
+        submission = next(
+            (sub for sub in assignment.get('submissions', []) 
+             if sub['student_id'] == str(student_id)),
+            None
+        )
+        if not submission:
+            return None
+
+        # Default rubric for assignment evaluation
+        default_rubric = """
+        1. Understanding & Implementation (1-4):
+           - Demonstrates understanding of assignment requirements
+           - Implements required components correctly
+           - Shows attention to detail
+           
+        2. Quality & Completeness (1-4):
+           - Work is complete and thorough
+           - Meets all assignment objectives
+           - Shows evidence of effort and care
+           
+        3. Presentation & Organization (1-4):
+           - Clear and professional presentation
+           - Well-structured and organized
+           - Follows required format and guidelines
+        """
+
+        # Initialize OpenAI client
+        client = OpenAI(api_key=os.getenv('OPENAI_KEY'))
+
+        # Create evaluation prompt
+        prompt_template = f"""As an assignment evaluator, assess this student's submission based on the provided rubric criteria. Follow these guidelines:
+
+        1. Evaluation Process:
+        - Use each rubric criterion (scored 1-4)
+        - Evaluate completeness and quality
+        - Check alignment with assignment requirements
+        - Calculate final score: sum of criteria scores converted to 10-point scale
+
+        Assignment Title: {assignment['title']}
+        Due Date: {assignment['due_date']}
+        
+        Submission Content:
+        {submission.get('text_content', 'No text content available')}
+
+        Rubric Criteria:
+        {default_rubric}
+
+        Provide your assessment in the following format:
+
+        **Overall Score and Summary**
+        - Score: [X]/10
+        - Overall Assessment: [2-3 sentence summary]
+
+        **Strengths**
+        - [Key strength 1]
+        - [Key strength 2]
+        - [Key strength 3]
+
+        **Areas for Improvement**
+        - [Improvement point 1]
+        - [Improvement point 2]
+        - [Improvement point 3]
+
+        **Specific Recommendations**
+        [2-3 sentences with actionable feedback]
+        """
+
+        # Generate evaluation using OpenAI
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": prompt_template}],
+            max_tokens=1000,
+            temperature=0.4
+        )
+
+        # Store evaluation in MongoDB
+        evaluation_doc = {
+            "assignment_id": assignment_id,
+            "student_id": student_id,
+            "session_id": session_id,
+            "evaluation": response.choices[0].message.content,
+            "evaluated_at": datetime.utcnow()
+        }
+        
+        assignment_evaluation_collection.insert_one(evaluation_doc)
+        return evaluation_doc
+
+    except Exception as e:
+        print(f"Error in evaluate_assignment: {str(e)}")
+        return None
+
+def display_evaluation_to_faculty(session_id, student_id, course_id):
+    """
+    Display interface for faculty to generate and view assignment evaluations
+    """
+    st.header("Evaluate Assignments")
+
+    try:
+        # Fetch available assignments
+        assignments = list(assignments_collection.find({
+            "session_id": str(session_id),
+            "course_id": course_id
+        }))
+
+        if not assignments:
+            st.info("No assignments found for this session.")
+            return
+
+        # Select assignment
+        assignment_options = {
+            f"{assignment['title']} (Due: {assignment['due_date'].strftime('%Y-%m-%d')})" if 'due_date' in assignment else assignment['title']: assignment['_id'] 
+            for assignment in assignments
+        }
+        
+        if assignment_options:
+            selected_assignment = st.selectbox(
+                "Select Assignment to Evaluate",
+                options=list(assignment_options.keys())
+            )
+
+            if selected_assignment:
+                assignment_id = assignment_options[selected_assignment]
+                assignment = assignments_collection.find_one({"_id": assignment_id})
+
+                if assignment:
+                    submissions = assignment.get('submissions', [])
+                    if not submissions:
+                        st.warning("No submissions found for this assignment.")
+                        return
+
+                    # Create a dropdown for student submissions
+                    student_options = {
+                        f"{students_collection.find_one({'_id': ObjectId(sub['student_id'])})['full_name']} (Submitted: {sub['submitted_at'].strftime('%Y-%m-%d %H:%M')})": sub['student_id']
+                        for sub in submissions
+                    }
+
+                    selected_student = st.selectbox(
+                        "Select Student Submission",
+                        options=list(student_options.keys())
+                    )
+
+                    if selected_student:
+                        student_id = student_options[selected_student]
+                        submission = next(sub for sub in submissions if sub['student_id'] == student_id)
+
+                        # Display submission details
+                        st.subheader("Submission Details")
+                        st.markdown(f"**Submitted:** {submission['submitted_at'].strftime('%Y-%m-%d %H:%M')}")
+                        st.markdown(f"**File Name:** {submission['file_name']}")
+                        
+                        # Add download button for submitted file
+                        if 'file_content' in submission:
+                            st.download_button(
+                                label="Download Submission",
+                                data=submission['file_content'],
+                                file_name=submission['file_name'],
+                                mime=submission['file_type']
+                            )
+
+                        # Check for existing evaluation
+                        existing_eval = assignment_evaluation_collection.find_one({
+                            "assignment_id": assignment_id,
+                            "student_id": student_id,
+                            "session_id": str(session_id)
+                        })
+
+                        if existing_eval:
+                            st.subheader("Evaluation Results")
+                            st.markdown(existing_eval['evaluation'])
+                            st.success("✓ Evaluation completed")
+                            
+                            if st.button("Regenerate Evaluation"):
+                                with st.spinner("Regenerating evaluation..."):
+                                    evaluation = evaluate_assignment(
+                                        str(session_id),
+                                        student_id,
+                                        assignment_id
+                                    )
+                                    if evaluation:
+                                        st.success("Evaluation regenerated successfully!")
+                                        st.rerun()
+                                    else:
+                                        st.error("Error regenerating evaluation.")
+                        else:
+                            if st.button("Generate Evaluation"):
+                                with st.spinner("Generating evaluation..."):
+                                    evaluation = evaluate_assignment(
+                                        str(session_id),
+                                        student_id,
+                                        assignment_id
+                                    )
+                                    if evaluation:
+                                        st.success("Evaluation generated successfully!")
+                                        st.markdown("### Generated Evaluation")
+                                        st.markdown(evaluation['evaluation'])
+                                        st.rerun()
+                                    else:
+                                        st.error("Error generating evaluation.")
+
+    except Exception as e:
+        st.error(f"An error occurred while loading the evaluations: {str(e)}")
+        print(f"Error in display_evaluation_to_faculty: {str(e)}")
+
+def display_assignment_results(assignment_id, student_id):
+    """
+    Display assignment results and analysis for a student
+    """
+    try:
+        # Fetch analysis from evaluation collection
+        analysis = assignment_evaluation_collection.find_one({
+            "assignment_id": assignment_id,
+            "student_id": str(student_id)
+        })
+        
+        if not analysis:
+            st.info("Evaluation will be available soon. Please check back later.")
+            return
+            
+        st.header("Assignment Evaluation")
+        
+        # Display evaluation content
+        st.markdown(analysis["evaluation"])
+        
+        # Display evaluation timestamp
+        st.caption(f"Evaluation generated on: {analysis['evaluated_at'].strftime('%Y-%m-%d %H:%M:%S UTC')}")
+            
+    except Exception as e:
+        st.error("An error occurred while loading the evaluation. Please try again later.")
+        print(f"Error in display_assignment_results: {str(e)}")

chatbot.py

@@ -1,67 +1,67 @@
-import streamlit as st
-import datetime
-from db import courses_collection2, faculty_collection, students_collection, vectors_collection, chat_history_collection
-from PIL import Image
-from dotenv import load_dotenv
-import os
-from datetime import datetime
-from bson import ObjectId
-from file_upload_vectorize import model
-from gen_mcqs import generate_mcqs, quizzes_collection
-
-load_dotenv()
-MONGO_URI = os.getenv('MONGO_URI')
-OPENAI_KEY = os.getenv('OPENAI_KEY')
-GEMINI_KEY = os.getenv('GEMINI_KEY')
-
-def insert_chat_message(user_id, session_id, role, content):
-    message = {
-        "role": role,
-        "content": content,
-        "timestamp": datetime.utcnow()
-    }
-    
-    chat_history_collection.update_one(
-        {"user_id": ObjectId(user_id), "session_id": session_id},
-        {"$push": {"messages": message}, "$set": {"timestamp": datetime.utcnow()}},
-        upsert=True
-    )
-
-def give_chat_response(user_id, session_id, question, title, description, context):
-    context_prompt = f"""
-    Based on the following session title, description, and context, answer the user's question in 3-4 lines:
-    
-    Title: {title}
-    Description: {description}
-    Context: {context}
-    
-    Question: {question}
-    
-    Please provide a clear and concise answer based on the information provided.
-    """
-    
-    response = model.generate_content(context_prompt)
-    if not response or not response.text:
-        return "No response received from the model"
-    
-    assistant_response = response.text.strip()
-    
-    # Save the chat message
-    insert_chat_message(user_id, session_id, "assistant", assistant_response)
-    
-    return assistant_response
-
-def create_quiz_by_context(user_id, session_id, context, length, session_title, session_description):
-    """Create a quiz based on the context provided"""
-    quiz = generate_mcqs(context, length, session_title, session_description)
-    if not quiz:
-        return "No quiz generated";
-    
-    # Save the quiz
-    quizzes_collection.insert_one({
-        "user_id": ObjectId(user_id),
-        "session_id": ObjectId(session_id),
-        "questions": quiz,
-        "timestamp": datetime.utcnow()
-    })
+import streamlit as st
+import datetime
+from db import courses_collection2, faculty_collection, students_collection, vectors_collection, chat_history_collection
+from PIL import Image
+from dotenv import load_dotenv
+import os
+from datetime import datetime
+from bson import ObjectId
+from file_upload_vectorize import model
+from gen_mcqs import generate_mcqs, quizzes_collection
+
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+OPENAI_KEY = os.getenv('OPENAI_KEY')
+GEMINI_KEY = os.getenv('GEMINI_KEY')
+
+def insert_chat_message(user_id, session_id, role, content):
+    message = {
+        "role": role,
+        "content": content,
+        "timestamp": datetime.utcnow()
+    }
+    
+    chat_history_collection.update_one(
+        {"user_id": ObjectId(user_id), "session_id": session_id},
+        {"$push": {"messages": message}, "$set": {"timestamp": datetime.utcnow()}},
+        upsert=True
+    )
+
+def give_chat_response(user_id, session_id, question, title, description, context):
+    context_prompt = f"""
+    Based on the following session title, description, and context, answer the user's question in 3-4 lines:
+    
+    Title: {title}
+    Description: {description}
+    Context: {context}
+    
+    Question: {question}
+    
+    Please provide a clear and concise answer based on the information provided.
+    """
+    
+    response = model.generate_content(context_prompt)
+    if not response or not response.text:
+        return "No response received from the model"
+    
+    assistant_response = response.text.strip()
+    
+    # Save the chat message
+    insert_chat_message(user_id, session_id, "assistant", assistant_response)
+    
+    return assistant_response
+
+def create_quiz_by_context(user_id, session_id, context, length, session_title, session_description):
+    """Create a quiz based on the context provided"""
+    quiz = generate_mcqs(context, length, session_title, session_description)
+    if not quiz:
+        return "No quiz generated";
+    
+    # Save the quiz
+    quizzes_collection.insert_one({
+        "user_id": ObjectId(user_id),
+        "session_id": ObjectId(session_id),
+        "questions": quiz,
+        "timestamp": datetime.utcnow()
+    })
     return "Quiz created successfully"

create_course.py

@@ -1,272 +1,272 @@
-from datetime import datetime, timedelta
-import os
-from typing import Dict, List, Any
-from pymongo import MongoClient
-import requests
-import uuid
-import openai
-from openai import OpenAI
-import streamlit as st
-from bson import ObjectId
-from dotenv import load_dotenv
-import json
-
-load_dotenv()
-MONGODB_URI = os.getenv("MONGO_URI")
-PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
-OPENAI_API_KEY = os.getenv("OPENAI_KEY")
-
-client = MongoClient(MONGODB_URI)
-db = client['novascholar_db']
-courses_collection = db['courses']
-
-def generate_perplexity_response(api_key, course_name):
-        headers = {
-            "accept": "application/json",
-            "content-type": "application/json",
-            "authorization": f"Bearer {api_key}"
-        }
-        
-        prompt = f"""
-        You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to generate comprehensive, academically rigorous course structures for undergraduate level education.
-
-        Please generate a detailed course structure for the course {course_name} in JSON format following these specifications:
-
-        1. The course structure should be appropriate for a full semester (14-16 weeks)
-        2. Each module should be designed for 2-4 weeks of instruction
-        3. Follow standard academic practices and nomenclature
-        4. Ensure progressive complexity from foundational to advanced concepts
-        5. The course_title should exactly match the course name provided in the prompt. No additional information should be included in the course_title field.
-        6: Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
-        7. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
-
-        
-        The JSON response should follow this structure:
-        {{
-            "course_title": "string",
-            "course_description": "string",
-            "modules": [
-                {{
-                    "module_title": "string",
-                    "sub_modules": [
-                        {{
-                            "title": "string",
-                            "topics": [string],
-                        }}
-                    ]
-                }}
-            ]
-        }}
-
-        Example response:
-        {{
-            "course_title": "Advanced Natural Language Processing",
-            "course_descriptio": "An advanced course covering modern approaches to NLP using deep learning, with focus on transformer architectures and their applications.",
-            "modules": [
-                {{
-                    "module_title": "Foundations of Modern NLP",
-                    "sub_modules": [
-                        {{
-                            "title": "Attention Mechanism",
-                            "topics": [
-                                "Self-attention",
-                                "Multi-head attention",
-                                "Positional encoding"
-                            ]
-                        }}
-                    ]
-                }}
-            ]
-        }}
-        """
-
-        messages = [
-            {
-                "role": "system",
-                "content": (
-                    "You are an expert educational AI assistant specializing in course design and curriculum planning. "
-                    "Your task is to generate accurate, detailed, and structured educational content for undergraduate-level and post-graduate-level courses. "
-                    "Provide detailed and accurate information tailored to the user's prompt."
-                    "Ensure that the responses are logical, follow standard academic practices, and include realistic concepts relevant to the course."
-                ),
-            },
-            {
-                "role": "user",
-                "content": prompt
-            },
-        ]
-        try:
-            client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-            response = client.chat.completions.create(
-                model="llama-3.1-sonar-small-128k-online",
-                messages=messages
-            )
-            content = response.choices[0].message.content
-            return content
-        except Exception as e:
-            st.error(f"Failed to fetch data from Perplexity API: {e}")
-            return ""
-
-def get_new_course_id():
-    """Generate a new course ID by incrementing the last course ID"""
-    last_course = courses_collection.find_one(sort=[("course_id", -1)])
-    if last_course:
-        last_course_id = int(last_course["course_id"][2:])
-        new_course_id = f"CS{last_course_id + 1}"
-    else:
-        new_course_id = "CS101"
-    return new_course_id
-
-
-def create_course(course_name, start_date, duration_weeks):
-        # Generate course overview
-        # overview_prompt = f"""Generate an overview for the undergraduate course {course_name} 
-        # Include all relevant concepts and key topics covered in a typical curriculum. 
-        # The response should be concise (300-400 words). Ensure that your response is in a valid JSON format."""
-        
-        # overview_prompt2 = f"""Generate an overview for the undergraduate course {course_name}. 
-        #                     The overview should include:
-        #                     The course title, a detailed course description, 
-        #                     a division of all relevant concepts and key topics into 4-6 logical modules, 
-        #                     capturing the flow and structure of a typical curriculum.
-        #                     Ensure the response adheres to the following JSON format:
-        #                         {{
-        #                             'overview': 'string',
-        #                             'modules': [
-        #                                 {{
-        #                                     'name': 'string',
-        #                                     'description': 'string'
-        #                                 }}
-        #                             ]
-        #                         }}
-        #                     overview: A detailed description of the course.
-        #                     modules: An array of 4-6 objects, each representing a logical module with a name and a brief description
-        #                     **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}"""
-        
-        # course_overview = generate_perplexity_response(PERPLEXITY_API_KEY, overview_prompt2)
-        # # print(course_overview)
-        # course_overview_store = course_overview
-        # # print(course_overview_store)
-        # # Generate modules
-        # # modules_prompt = f"Based on this overview: {course_overview}\nCreate 4-6 logical modules for the course, each module should group related concepts and each module may include reference books if applicable"
-        # sub_modules_prompt = f"""Using the provided modules in the overview {course_overview_store}, generate 2-3 submodules for each module. 
-        #                         Each submodule should represent a cohesive subset of the module's topics, logically organized for teaching purposes.
-        #                         Ensure the response adheres to the following JSON format:
-        #                         {
-        #                             'modules': [
-        #                                 {
-        #                                     'name': 'string',
-        #                                     'sub_modules': [
-        #                                         {
-        #                                             'name': 'string',
-        #                                             'description': 'string'
-        #                                         }
-        #                                     ]
-        #                                 }
-        #                             ]
-        #                         }
-        #                         modules: An array where each object contains the name of the module and its corresponding sub_modules.
-        #                         sub_modules: An array of 2-3 objects for each module, each having a name and a brief description."
-        #                         **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}
-        #                     """
-        # sub_modules = generate_perplexity_response(PERPLEXITY_API_KEY, sub_modules_prompt)
-        
-        # # modules_response = generate_perplexity_response(modules_prompt)
-        # print(sub_modules)
-        
-        # total_sessions = duration_weeks * sessions_per_week
-
-        course_plan = generate_perplexity_response(PERPLEXITY_API_KEY, course_name)
-        course_plan_json = json.loads(course_plan)
-        
-        # Generate sessions for each module
-        all_sessions = []
-        for module in course_plan_json['modules']:
-            for sub_module in module['sub_modules']:
-                for topic in sub_module['topics']:
-                    session = create_session(
-                        title=topic,
-                        date=start_date,
-                        module_name=module['module_title']
-                    )
-                    # print(session)
-                    all_sessions.append(session)
-                    start_date += timedelta(days=7)  # Next session after a week
-
-        # sample_sessions = [
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def25'), 'title': 'What is Generative AI?', 'date': datetime(2024, 12, 22, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def26'), 'title': 'History and Evolution of AI', 'date': datetime(2024, 12, 29, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def27'), 'title': 'Types of Generative AI (e.g., GANs, VAEs, LLMs)', 'date': datetime(2025, 1, 5, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 505626), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def28'), 'title': 'Overview of popular GenAI tools (e.g., ChatGPT, Claude, Google Gemini)', 'date': datetime(2025, 1, 12, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 506559), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def29'), 'title': 'Frameworks for building GenAI models (e.g., TensorFlow, PyTorch)', 'date': datetime(2025, 1, 19, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 506559), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2a'), 'title': 'Integration with other AI technologies', 'date': datetime(2025, 1, 26, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 507612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2b'), 'title': 'Text-to-text models (e.g., GPT-3, BERT)', 'date': datetime(2025, 2, 2, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 508512), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2c'), 'title': 'Text generation for content creation and marketing', 'date': datetime(2025, 2, 9, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 508512), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2d'), 'title': 'Chatbots and conversational interfaces', 'date': datetime(2025, 2, 16, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 509612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2e'), 'title': 'Generative Adversarial Networks (GANs)', 'date': datetime(2025, 2, 23, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 509612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def2f'), 'title': 'Variational Autoencoders (VAEs)', 'date': datetime(2025, 3, 2, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 510612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def30'), 'title': 'Applications in art, design, and media', 'date': datetime(2025, 3, 9, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 511497), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def31'), 'title': 'Understanding prompt design principles', 'date': datetime(2025, 3, 16, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 511497), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def33'), 'title': 'Advanced techniques for fine-tuning models', 'date': datetime(2025, 3, 30, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 512514), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def34'), 'title': 'Ethical implications of AI-generated content', 'date': datetime(2025, 4, 6, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 513613), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def35'), 'title': 'Addressing bias in AI models', 'date': datetime(2025, 4, 13, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 514639), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def36'), 'title': 'Regulatory frameworks and guidelines', 'date': datetime(2025, 4, 20, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 514639), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def37'), 'title': 'Case studies from various industries (e.g., marketing, healthcare, finance)', 'date': datetime(2025, 4, 27, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 515610), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def38'), 'title': 'Success stories and challenges faced by companies using GenAI', 'date': datetime(2025, 5, 4, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 515610), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def39'), 'title': 'Guidelines for developing a GenAI project', 'date': datetime(2025, 5, 11, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 516614), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def3a'), 'title': 'Tools and resources for project implementation', 'date': datetime(2025, 5, 18, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 516614), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def3b'), 'title': 'Best practices for testing and deployment', 'date': datetime(2025, 5, 25, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 517563), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}}
-        # ]
-
-        # small_sample_sessions = [
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def25'), 'title': 'What is Generative AI?', 'date': datetime(2024, 12, 22, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        #     {'session_id': ObjectId('6767d0bbad8316ac358def26'), 'title': 'History and Evolution of AI', 'date': datetime(2024, 12, 29, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
-        # ]
-
-
-        # print(all_sessions)
-
-        print("Number of sessions:", len(all_sessions))
-        # Create course document
-        # course_description = course_plan_json['course_description']
-        # course_doc = {
-        #     "course_id": get_new_course_id(),
-        #     "title": course_name,
-        #     "description": course_description,
-        #     "faculty": faculty_name,
-        #     "faculty_id": faculty_id,
-        #     "duration": f"{duration_weeks} weeks",
-        #     "created_at": datetime.utcnow(),
-        #     "sessions": all_sessions
-        # }
-        # try:
-        #     courses_collection.insert_one(course_doc)
-        # except Exception as e:
-        #     st.error(f"Failed to insert course data into the database: {e}")
-
-        # print(course_plan)
-
-def create_session(title: str, date: datetime, module_name: str):
-        """Create a session document with pre-class, in-class, and post-class components."""
-        return {
-            "session_id": ObjectId(),
-            "title": title,
-            "date": date,
-            "status": "upcoming",
-            "created_at": datetime.utcnow(),
-            "pre_class": {
-                "resources": [],
-                "completion_required": True
-            },
-            "in_class": {
-                "quiz": [],
-                "polls": []
-            },
-            "post_class": {
-                "assignments": []
-            }
-        }
-
-# Usage example:
-if __name__ == "__main__":
+from datetime import datetime, timedelta
+import os
+from typing import Dict, List, Any
+from pymongo import MongoClient
+import requests
+import uuid
+import openai
+from openai import OpenAI
+import streamlit as st
+from bson import ObjectId
+from dotenv import load_dotenv
+import json
+
+load_dotenv()
+MONGODB_URI = os.getenv("MONGO_URI")
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_KEY")
+
+client = MongoClient(MONGODB_URI)
+db = client['novascholar_db']
+courses_collection = db['courses']
+
+def generate_perplexity_response(api_key, course_name):
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/json",
+            "authorization": f"Bearer {api_key}"
+        }
+        
+        prompt = f"""
+        You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to generate comprehensive, academically rigorous course structures for undergraduate level education.
+
+        Please generate a detailed course structure for the course {course_name} in JSON format following these specifications:
+
+        1. The course structure should be appropriate for a full semester (14-16 weeks)
+        2. Each module should be designed for 2-4 weeks of instruction
+        3. Follow standard academic practices and nomenclature
+        4. Ensure progressive complexity from foundational to advanced concepts
+        5. The course_title should exactly match the course name provided in the prompt. No additional information should be included in the course_title field.
+        6: Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
+        7. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+
+        
+        The JSON response should follow this structure:
+        {{
+            "course_title": "string",
+            "course_description": "string",
+            "modules": [
+                {{
+                    "module_title": "string",
+                    "sub_modules": [
+                        {{
+                            "title": "string",
+                            "topics": [string],
+                        }}
+                    ]
+                }}
+            ]
+        }}
+
+        Example response:
+        {{
+            "course_title": "Advanced Natural Language Processing",
+            "course_descriptio": "An advanced course covering modern approaches to NLP using deep learning, with focus on transformer architectures and their applications.",
+            "modules": [
+                {{
+                    "module_title": "Foundations of Modern NLP",
+                    "sub_modules": [
+                        {{
+                            "title": "Attention Mechanism",
+                            "topics": [
+                                "Self-attention",
+                                "Multi-head attention",
+                                "Positional encoding"
+                            ]
+                        }}
+                    ]
+                }}
+            ]
+        }}
+        """
+
+        messages = [
+            {
+                "role": "system",
+                "content": (
+                    "You are an expert educational AI assistant specializing in course design and curriculum planning. "
+                    "Your task is to generate accurate, detailed, and structured educational content for undergraduate-level and post-graduate-level courses. "
+                    "Provide detailed and accurate information tailored to the user's prompt."
+                    "Ensure that the responses are logical, follow standard academic practices, and include realistic concepts relevant to the course."
+                ),
+            },
+            {
+                "role": "user",
+                "content": prompt
+            },
+        ]
+        try:
+            client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
+            response = client.chat.completions.create(
+                model="llama-3.1-sonar-small-128k-online",
+                messages=messages
+            )
+            content = response.choices[0].message.content
+            return content
+        except Exception as e:
+            st.error(f"Failed to fetch data from Perplexity API: {e}")
+            return ""
+
+def get_new_course_id():
+    """Generate a new course ID by incrementing the last course ID"""
+    last_course = courses_collection.find_one(sort=[("course_id", -1)])
+    if last_course:
+        last_course_id = int(last_course["course_id"][2:])
+        new_course_id = f"CS{last_course_id + 1}"
+    else:
+        new_course_id = "CS101"
+    return new_course_id
+
+
+def create_course(course_name, start_date, duration_weeks):
+        # Generate course overview
+        # overview_prompt = f"""Generate an overview for the undergraduate course {course_name} 
+        # Include all relevant concepts and key topics covered in a typical curriculum. 
+        # The response should be concise (300-400 words). Ensure that your response is in a valid JSON format."""
+        
+        # overview_prompt2 = f"""Generate an overview for the undergraduate course {course_name}. 
+        #                     The overview should include:
+        #                     The course title, a detailed course description, 
+        #                     a division of all relevant concepts and key topics into 4-6 logical modules, 
+        #                     capturing the flow and structure of a typical curriculum.
+        #                     Ensure the response adheres to the following JSON format:
+        #                         {{
+        #                             'overview': 'string',
+        #                             'modules': [
+        #                                 {{
+        #                                     'name': 'string',
+        #                                     'description': 'string'
+        #                                 }}
+        #                             ]
+        #                         }}
+        #                     overview: A detailed description of the course.
+        #                     modules: An array of 4-6 objects, each representing a logical module with a name and a brief description
+        #                     **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}"""
+        
+        # course_overview = generate_perplexity_response(PERPLEXITY_API_KEY, overview_prompt2)
+        # # print(course_overview)
+        # course_overview_store = course_overview
+        # # print(course_overview_store)
+        # # Generate modules
+        # # modules_prompt = f"Based on this overview: {course_overview}\nCreate 4-6 logical modules for the course, each module should group related concepts and each module may include reference books if applicable"
+        # sub_modules_prompt = f"""Using the provided modules in the overview {course_overview_store}, generate 2-3 submodules for each module. 
+        #                         Each submodule should represent a cohesive subset of the module's topics, logically organized for teaching purposes.
+        #                         Ensure the response adheres to the following JSON format:
+        #                         {
+        #                             'modules': [
+        #                                 {
+        #                                     'name': 'string',
+        #                                     'sub_modules': [
+        #                                         {
+        #                                             'name': 'string',
+        #                                             'description': 'string'
+        #                                         }
+        #                                     ]
+        #                                 }
+        #                             ]
+        #                         }
+        #                         modules: An array where each object contains the name of the module and its corresponding sub_modules.
+        #                         sub_modules: An array of 2-3 objects for each module, each having a name and a brief description."
+        #                         **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}
+        #                     """
+        # sub_modules = generate_perplexity_response(PERPLEXITY_API_KEY, sub_modules_prompt)
+        
+        # # modules_response = generate_perplexity_response(modules_prompt)
+        # print(sub_modules)
+        
+        # total_sessions = duration_weeks * sessions_per_week
+
+        course_plan = generate_perplexity_response(PERPLEXITY_API_KEY, course_name)
+        course_plan_json = json.loads(course_plan)
+        
+        # Generate sessions for each module
+        all_sessions = []
+        for module in course_plan_json['modules']:
+            for sub_module in module['sub_modules']:
+                for topic in sub_module['topics']:
+                    session = create_session(
+                        title=topic,
+                        date=start_date,
+                        module_name=module['module_title']
+                    )
+                    # print(session)
+                    all_sessions.append(session)
+                    start_date += timedelta(days=7)  # Next session after a week
+
+        # sample_sessions = [
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def25'), 'title': 'What is Generative AI?', 'date': datetime(2024, 12, 22, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def26'), 'title': 'History and Evolution of AI', 'date': datetime(2024, 12, 29, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def27'), 'title': 'Types of Generative AI (e.g., GANs, VAEs, LLMs)', 'date': datetime(2025, 1, 5, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 505626), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def28'), 'title': 'Overview of popular GenAI tools (e.g., ChatGPT, Claude, Google Gemini)', 'date': datetime(2025, 1, 12, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 506559), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def29'), 'title': 'Frameworks for building GenAI models (e.g., TensorFlow, PyTorch)', 'date': datetime(2025, 1, 19, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 506559), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2a'), 'title': 'Integration with other AI technologies', 'date': datetime(2025, 1, 26, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 507612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2b'), 'title': 'Text-to-text models (e.g., GPT-3, BERT)', 'date': datetime(2025, 2, 2, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 508512), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2c'), 'title': 'Text generation for content creation and marketing', 'date': datetime(2025, 2, 9, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 508512), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2d'), 'title': 'Chatbots and conversational interfaces', 'date': datetime(2025, 2, 16, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 509612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2e'), 'title': 'Generative Adversarial Networks (GANs)', 'date': datetime(2025, 2, 23, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 509612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def2f'), 'title': 'Variational Autoencoders (VAEs)', 'date': datetime(2025, 3, 2, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 510612), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def30'), 'title': 'Applications in art, design, and media', 'date': datetime(2025, 3, 9, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 511497), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def31'), 'title': 'Understanding prompt design principles', 'date': datetime(2025, 3, 16, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 511497), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def33'), 'title': 'Advanced techniques for fine-tuning models', 'date': datetime(2025, 3, 30, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 512514), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def34'), 'title': 'Ethical implications of AI-generated content', 'date': datetime(2025, 4, 6, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 513613), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def35'), 'title': 'Addressing bias in AI models', 'date': datetime(2025, 4, 13, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 514639), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def36'), 'title': 'Regulatory frameworks and guidelines', 'date': datetime(2025, 4, 20, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 514639), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def37'), 'title': 'Case studies from various industries (e.g., marketing, healthcare, finance)', 'date': datetime(2025, 4, 27, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 515610), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def38'), 'title': 'Success stories and challenges faced by companies using GenAI', 'date': datetime(2025, 5, 4, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 515610), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def39'), 'title': 'Guidelines for developing a GenAI project', 'date': datetime(2025, 5, 11, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 516614), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def3a'), 'title': 'Tools and resources for project implementation', 'date': datetime(2025, 5, 18, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 516614), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def3b'), 'title': 'Best practices for testing and deployment', 'date': datetime(2025, 5, 25, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 517563), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}}
+        # ]
+
+        # small_sample_sessions = [
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def25'), 'title': 'What is Generative AI?', 'date': datetime(2024, 12, 22, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        #     {'session_id': ObjectId('6767d0bbad8316ac358def26'), 'title': 'History and Evolution of AI', 'date': datetime(2024, 12, 29, 14, 11, 27, 153899), 'status': 'upcoming', 'created_at': datetime(2024, 12, 22, 8, 41, 31, 504599), 'pre_class': {'resources': [], 'completion_required': True}, 'in_class': {'quiz': [], 'polls': []}, 'post_class': {'assignments': []}},
+        # ]
+
+
+        # print(all_sessions)
+
+        print("Number of sessions:", len(all_sessions))
+        # Create course document
+        # course_description = course_plan_json['course_description']
+        # course_doc = {
+        #     "course_id": get_new_course_id(),
+        #     "title": course_name,
+        #     "description": course_description,
+        #     "faculty": faculty_name,
+        #     "faculty_id": faculty_id,
+        #     "duration": f"{duration_weeks} weeks",
+        #     "created_at": datetime.utcnow(),
+        #     "sessions": all_sessions
+        # }
+        # try:
+        #     courses_collection.insert_one(course_doc)
+        # except Exception as e:
+        #     st.error(f"Failed to insert course data into the database: {e}")
+
+        # print(course_plan)
+
+def create_session(title: str, date: datetime, module_name: str):
+        """Create a session document with pre-class, in-class, and post-class components."""
+        return {
+            "session_id": ObjectId(),
+            "title": title,
+            "date": date,
+            "status": "upcoming",
+            "created_at": datetime.utcnow(),
+            "pre_class": {
+                "resources": [],
+                "completion_required": True
+            },
+            "in_class": {
+                "quiz": [],
+                "polls": []
+            },
+            "post_class": {
+                "assignments": []
+            }
+        }
+
+# Usage example:
+if __name__ == "__main__":
     create_course("Introduction to Data Analytics", datetime.now(), 2)

create_course2.py

@@ -1,331 +1,331 @@
-from datetime import datetime, timedelta
-import os
-from typing import Dict, List, Any
-from pymongo import MongoClient
-import requests
-import uuid
-import openai
-from openai import OpenAI
-import streamlit as st
-from bson import ObjectId
-from dotenv import load_dotenv
-import json
-
-load_dotenv()
-MONGODB_URI = os.getenv("MONGO_URI")
-PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
-OPENAI_API_KEY = os.getenv("OPENAI_KEY")
-
-client = MongoClient(MONGODB_URI)
-db = client['novascholar_db']
-courses_collection = db['courses']
-
-def generate_perplexity_response(api_key, course_name, duration_weeks, sessions_per_week):
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": f"Bearer {api_key}"
-    }
-    
-    # Calculate sessions based on duration
-    total_sessions = duration_weeks * sessions_per_week  # Assuming 2 sessions per week
-    
-    prompt = f"""
-    You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to generate a comprehensive, academically rigorous course structure for the course {course_name} that fits exactly within {duration_weeks} weeks with {total_sessions} total sessions ({sessions_per_week} sessions per week).
-
-    Please generate a detailed course structure in JSON format following these specifications:
-
-    1. The course structure must be designed for exactly {duration_weeks} weeks with {total_sessions} total sessions
-    2. Each module should contain an appropriate number of sessions that sum up to exactly {total_sessions}
-    3. Each session should be designed for a 1-1.5-hour class duration
-    4. Follow standard academic practices and nomenclature
-    5. Ensure progressive complexity from foundational to advanced concepts
-    6. The course_title should exactly match the course name provided
-    7. Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
-    8. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
-
-    The JSON response should follow this structure:
-    {{
-        "course_title": "string",
-        "course_description": "string",
-        "total_duration_weeks": {duration_weeks},
-        "sessions_per_week": {sessions_per_week},
-        "total_sessions": {total_sessions},
-        "modules": [
-            {{
-                "module_title": "string",
-                "module_duration_sessions": number,
-                "sub_modules": [
-                    {{
-                        "title": "string",
-                        "topics": [
-                            {{
-                                "title": "string",
-                                "short_description": "string",
-                                "concise_learning_objectives": ["string"]
-                            }}
-                        ]
-                    }}
-                ]
-            }}
-        ]
-    }}
-
-    Ensure that:
-    1. The sum of all module_duration_sessions equals exactly {total_sessions}
-    2. Each topic has clear learning objectives
-    3. Topics build upon each other logically
-    4. Content is distributed evenly across the available sessions
-    5. **This Instruction is Strictly followed: **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.****
-
-    """
-
-    messages = [
-        {
-            "role": "system",
-            "content": (
-                "You are an expert educational AI assistant specializing in course design and curriculum planning. "
-                "Your task is to generate accurate, detailed, and structured educational content that precisely fits "
-                "the specified duration."
-            ),
-        },
-        {
-            "role": "user",
-            "content": prompt
-        },
-    ]
-    
-    try:
-        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-        response = client.chat.completions.create(
-            model="llama-3.1-sonar-small-128k-online",
-            messages=messages
-        )
-        content = response.choices[0].message.content
-        
-        # Validate session count
-        course_plan = json.loads(content)
-        total_planned_sessions = sum(
-            module.get('module_duration_sessions', 0) 
-            for module in course_plan.get('modules', [])
-        )
-        
-        if abs(total_planned_sessions - total_sessions) > 5:
-            raise ValueError(f"Generated plan has {total_planned_sessions} sessions, but {total_sessions} were requested")
-            
-        return content
-    except Exception as e:
-        st.error(f"Failed to fetch data from Perplexity API: {e}")
-        return ""
-
-def generate_session_resources(api_key, session_titles: List[str]):
-    """
-    Generate relevant resources for each session title separately
-    """
-    resources_prompt = f"""
-    You are an expert educational content curator. For each session title provided, suggest highly relevant and accurate learning resources.
-    Please provide resources for these sessions: {session_titles}
-
-    For each session, provide resources in this JSON format:
-    {{
-        "session_resources": [
-            {{
-                "session_title": "string",
-                "resources": {{
-                    "readings": [
-                        {{
-                            "title": "string",
-                            "url": "string",
-                            "type": "string",
-                            "estimated_read_time": "string"
-                        }}
-                    ],
-                    "books": [
-                        {{
-                            "title": "string",
-                            "author": "string",
-                            "isbn": "string",
-                            "chapters": "string"
-                        }}
-                    ],
-                    "additional_resources": [
-                        {{
-                            "title": "string",
-                            "url": "string",
-                            "type": "string",
-                            "description": "string"
-                        }}
-                    ]
-                }}
-            }}
-        ]
-    }}
-
-    Guidelines:
-    1. Ensure all URLs are real and currently active
-    2. Prioritize high-quality, authoritative sources
-    3. Include 1-2 resources of each type
-    5. For readings, include a mix of academic and practical resources. It can exceed to 3-4 readings 
-    6. Book references should be real, recently published works
-    7. Additional resources can include tools, documentation, or practice platforms
-    8. Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
-    9. ***NOTE: **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
-    """
-
-    messages = [
-        {
-            "role": "system",
-            "content": "You are an expert educational content curator, focused on providing accurate and relevant learning resources.",
-        },
-        {
-            "role": "user",
-            "content": resources_prompt
-        },
-    ]
-
-    try:
-        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-        response = client.chat.completions.create(
-            model="llama-3.1-sonar-small-128k-online",
-            messages=messages
-        )
-        print("Response is: \n", response.choices[0].message.content)
-        # try:
-        #     return json.loads(response.choices[0].message.content)
-        # except json.JSONDecodeError as e:
-        #     st.error(f"Failed to decode JSON response: {e}")
-        #     return None
-        return response.choices[0].message.content
-    except Exception as e:
-        st.error(f"Failed to generate resources: {e}")
-        return None
-
-def validate_course_plan(course_plan):
-    required_fields = ['course_title', 'course_description', 'modules']
-    if not all(field in course_plan for field in required_fields):
-        raise ValueError("Invalid course plan structure")
-    
-    for module in course_plan['modules']:
-        if 'module_title' not in module or 'sub_modules' not in module:
-            raise ValueError("Invalid module structure")
-
-def create_session(title: str, date: datetime, module_name: str, resources: dict):
-    """Create a session document with pre-class, in-class, and post-class components."""
-    return {
-        "session_id": ObjectId(),
-        "title": title,
-        "date": date,
-        "status": "upcoming",
-        "created_at": datetime.utcnow(),
-        "module_name": module_name,
-        "pre_class": {
-            "resources": [],
-            "completion_required": True
-        },
-        "in_class": {
-            "quiz": [],
-            "polls": []
-        },
-        "post_class": {
-            "assignments": []
-        },
-        "external_resources": {
-            "readings": resources.get("readings", []),
-            "books": resources.get("books", []),
-            "additional_resources": resources.get("additional_resources", [])
-        }
-    }
-
-def create_course(course_name: str, start_date: datetime, duration_weeks: int, sessions_per_week: int):
-    # First generate a course plan using Perplexity API
-    # course_plan = generate_perplexity_response(PERPLEXITY_API_KEY, course_name, duration_weeks, sessions_per_week)
-    # course_plan_json = json.loads(course_plan)
-    
-    # print("Course Structure is: \n", course_plan_json);
-
-    # Earlier Code: 
-    # Generate sessions for each module with resources
-    # all_sessions = []
-    # current_date = start_date
-    
-    # for module in course_plan_json['modules']:
-    #     for sub_module in module['sub_modules']:
-    #         for topic in sub_module['topics']:
-    #             session = create_session(
-    #                 title=topic['title'],
-    #                 date=current_date,
-    #                 module_name=module['module_title'],
-    #                 resources=topic['resources']
-    #             )
-    #             all_sessions.append(session)
-    #             current_date += timedelta(days=3.5)  # Spacing sessions evenly across the week
-    
-    # return course_plan_json, all_sessions
-
-    # New Code:
-    # Extract all session titles
-    session_titles = []
-    # Load the course plan JSON
-    course_plan_json = {}
-    with open('sample_files/sample_course.json', 'r') as file:
-        course_plan_json = json.load(file)
-
-    for module in course_plan_json['modules']:
-        for sub_module in module['sub_modules']:
-            for topic in sub_module['topics']:
-                session_titles.append(topic['title'])
-    
-    # Generate resources for all sessions
-    session_resources = generate_session_resources(PERPLEXITY_API_KEY, session_titles)
-    # print("Session Resources are: \n", session_resources)
-    resources = json.loads(session_resources)
-    # print("Resources JSON is: \n", resources_json)
-    
-    # print("Session Resources are: \n", session_resources)
-
-    # Create a mapping of session titles to their resources
-    
-    # Import Resources JSON
-    # resources = {}
-    # with open('sample_files/sample_course_resources.json', 'r') as file:
-    #     resources = json.load(file)
-
-    resources_map = {
-        resource['session_title']: resource['resources']
-        for resource in resources['session_resources']
-    }
-    print("Resources Map is: \n", resources_map)
-    # print("Sample is: ", resources_map.get('Overview of ML Concepts, History, and Applications'));
-    # Generate sessions with their corresponding resources
-    all_sessions = []
-    current_date = start_date
-    
-    for module in course_plan_json['modules']:
-        for sub_module in module['sub_modules']:
-            for topic in sub_module['topics']:
-                session = create_session(
-                    title=topic['title'],
-                    date=current_date,
-                    module_name=module['module_title'],
-                    resources=resources_map.get(topic['title'], {})
-                )
-                all_sessions.append(session)
-                current_date += timedelta(days=3.5)
-    
-    print("All Sessions are: \n", all_sessions)
-
-def get_new_course_id():
-    """Generate a new course ID by incrementing the last course ID"""
-    last_course = courses_collection.find_one(sort=[("course_id", -1)])
-    if last_course:
-        last_course_id = int(last_course["course_id"][2:])
-        new_course_id = f"CS{last_course_id + 1}"
-    else:
-        new_course_id = "CS101"
-    return new_course_id
-
-# if __name__ == "__main__":
-#     course_name = "Introduction to Machine Learning"
-#     start_date = datetime(2022, 9, 1)
-#     duration_weeks = 4
+from datetime import datetime, timedelta
+import os
+from typing import Dict, List, Any
+from pymongo import MongoClient
+import requests
+import uuid
+import openai
+from openai import OpenAI
+import streamlit as st
+from bson import ObjectId
+from dotenv import load_dotenv
+import json
+
+load_dotenv()
+MONGODB_URI = os.getenv("MONGO_URI")
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_KEY")
+
+client = MongoClient(MONGODB_URI)
+db = client['novascholar_db']
+courses_collection = db['courses']
+
+def generate_perplexity_response(api_key, course_name, duration_weeks, sessions_per_week):
+    headers = {
+        "accept": "application/json",
+        "content-type": "application/json",
+        "authorization": f"Bearer {api_key}"
+    }
+    
+    # Calculate sessions based on duration
+    total_sessions = duration_weeks * sessions_per_week  # Assuming 2 sessions per week
+    
+    prompt = f"""
+    You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to generate a comprehensive, academically rigorous course structure for the course {course_name} that fits exactly within {duration_weeks} weeks with {total_sessions} total sessions ({sessions_per_week} sessions per week).
+
+    Please generate a detailed course structure in JSON format following these specifications:
+
+    1. The course structure must be designed for exactly {duration_weeks} weeks with {total_sessions} total sessions
+    2. Each module should contain an appropriate number of sessions that sum up to exactly {total_sessions}
+    3. Each session should be designed for a 1-1.5-hour class duration
+    4. Follow standard academic practices and nomenclature
+    5. Ensure progressive complexity from foundational to advanced concepts
+    6. The course_title should exactly match the course name provided
+    7. Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
+    8. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+
+    The JSON response should follow this structure:
+    {{
+        "course_title": "string",
+        "course_description": "string",
+        "total_duration_weeks": {duration_weeks},
+        "sessions_per_week": {sessions_per_week},
+        "total_sessions": {total_sessions},
+        "modules": [
+            {{
+                "module_title": "string",
+                "module_duration_sessions": number,
+                "sub_modules": [
+                    {{
+                        "title": "string",
+                        "topics": [
+                            {{
+                                "title": "string",
+                                "short_description": "string",
+                                "concise_learning_objectives": ["string"]
+                            }}
+                        ]
+                    }}
+                ]
+            }}
+        ]
+    }}
+
+    Ensure that:
+    1. The sum of all module_duration_sessions equals exactly {total_sessions}
+    2. Each topic has clear learning objectives
+    3. Topics build upon each other logically
+    4. Content is distributed evenly across the available sessions
+    5. **This Instruction is Strictly followed: **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.****
+
+    """
+
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are an expert educational AI assistant specializing in course design and curriculum planning. "
+                "Your task is to generate accurate, detailed, and structured educational content that precisely fits "
+                "the specified duration."
+            ),
+        },
+        {
+            "role": "user",
+            "content": prompt
+        },
+    ]
+    
+    try:
+        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
+        response = client.chat.completions.create(
+            model="llama-3.1-sonar-small-128k-online",
+            messages=messages
+        )
+        content = response.choices[0].message.content
+        
+        # Validate session count
+        course_plan = json.loads(content)
+        total_planned_sessions = sum(
+            module.get('module_duration_sessions', 0) 
+            for module in course_plan.get('modules', [])
+        )
+        
+        if abs(total_planned_sessions - total_sessions) > 5:
+            raise ValueError(f"Generated plan has {total_planned_sessions} sessions, but {total_sessions} were requested")
+            
+        return content
+    except Exception as e:
+        st.error(f"Failed to fetch data from Perplexity API: {e}")
+        return ""
+
+def generate_session_resources(api_key, session_titles: List[str]):
+    """
+    Generate relevant resources for each session title separately
+    """
+    resources_prompt = f"""
+    You are an expert educational content curator. For each session title provided, suggest highly relevant and accurate learning resources.
+    Please provide resources for these sessions: {session_titles}
+
+    For each session, provide resources in this JSON format:
+    {{
+        "session_resources": [
+            {{
+                "session_title": "string",
+                "resources": {{
+                    "readings": [
+                        {{
+                            "title": "string",
+                            "url": "string",
+                            "type": "string",
+                            "estimated_read_time": "string"
+                        }}
+                    ],
+                    "books": [
+                        {{
+                            "title": "string",
+                            "author": "string",
+                            "isbn": "string",
+                            "chapters": "string"
+                        }}
+                    ],
+                    "additional_resources": [
+                        {{
+                            "title": "string",
+                            "url": "string",
+                            "type": "string",
+                            "description": "string"
+                        }}
+                    ]
+                }}
+            }}
+        ]
+    }}
+
+    Guidelines:
+    1. Ensure all URLs are real and currently active
+    2. Prioritize high-quality, authoritative sources
+    3. Include 1-2 resources of each type
+    5. For readings, include a mix of academic and practical resources. It can exceed to 3-4 readings 
+    6. Book references should be real, recently published works
+    7. Additional resources can include tools, documentation, or practice platforms
+    8. Ensure that the property names are enclosed in double quotes (") and followed by a colon (:), and the values are enclosed in double quotes (").
+    9. ***NOTE: **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+    """
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are an expert educational content curator, focused on providing accurate and relevant learning resources.",
+        },
+        {
+            "role": "user",
+            "content": resources_prompt
+        },
+    ]
+
+    try:
+        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
+        response = client.chat.completions.create(
+            model="llama-3.1-sonar-small-128k-online",
+            messages=messages
+        )
+        print("Response is: \n", response.choices[0].message.content)
+        # try:
+        #     return json.loads(response.choices[0].message.content)
+        # except json.JSONDecodeError as e:
+        #     st.error(f"Failed to decode JSON response: {e}")
+        #     return None
+        return response.choices[0].message.content
+    except Exception as e:
+        st.error(f"Failed to generate resources: {e}")
+        return None
+
+def validate_course_plan(course_plan):
+    required_fields = ['course_title', 'course_description', 'modules']
+    if not all(field in course_plan for field in required_fields):
+        raise ValueError("Invalid course plan structure")
+    
+    for module in course_plan['modules']:
+        if 'module_title' not in module or 'sub_modules' not in module:
+            raise ValueError("Invalid module structure")
+
+def create_session(title: str, date: datetime, module_name: str, resources: dict):
+    """Create a session document with pre-class, in-class, and post-class components."""
+    return {
+        "session_id": ObjectId(),
+        "title": title,
+        "date": date,
+        "status": "upcoming",
+        "created_at": datetime.utcnow(),
+        "module_name": module_name,
+        "pre_class": {
+            "resources": [],
+            "completion_required": True
+        },
+        "in_class": {
+            "quiz": [],
+            "polls": []
+        },
+        "post_class": {
+            "assignments": []
+        },
+        "external_resources": {
+            "readings": resources.get("readings", []),
+            "books": resources.get("books", []),
+            "additional_resources": resources.get("additional_resources", [])
+        }
+    }
+
+def create_course(course_name: str, start_date: datetime, duration_weeks: int, sessions_per_week: int):
+    # First generate a course plan using Perplexity API
+    # course_plan = generate_perplexity_response(PERPLEXITY_API_KEY, course_name, duration_weeks, sessions_per_week)
+    # course_plan_json = json.loads(course_plan)
+    
+    # print("Course Structure is: \n", course_plan_json);
+
+    # Earlier Code: 
+    # Generate sessions for each module with resources
+    # all_sessions = []
+    # current_date = start_date
+    
+    # for module in course_plan_json['modules']:
+    #     for sub_module in module['sub_modules']:
+    #         for topic in sub_module['topics']:
+    #             session = create_session(
+    #                 title=topic['title'],
+    #                 date=current_date,
+    #                 module_name=module['module_title'],
+    #                 resources=topic['resources']
+    #             )
+    #             all_sessions.append(session)
+    #             current_date += timedelta(days=3.5)  # Spacing sessions evenly across the week
+    
+    # return course_plan_json, all_sessions
+
+    # New Code:
+    # Extract all session titles
+    session_titles = []
+    # Load the course plan JSON
+    course_plan_json = {}
+    with open('sample_files/sample_course.json', 'r') as file:
+        course_plan_json = json.load(file)
+
+    for module in course_plan_json['modules']:
+        for sub_module in module['sub_modules']:
+            for topic in sub_module['topics']:
+                session_titles.append(topic['title'])
+    
+    # Generate resources for all sessions
+    session_resources = generate_session_resources(PERPLEXITY_API_KEY, session_titles)
+    # print("Session Resources are: \n", session_resources)
+    resources = json.loads(session_resources)
+    # print("Resources JSON is: \n", resources_json)
+    
+    # print("Session Resources are: \n", session_resources)
+
+    # Create a mapping of session titles to their resources
+    
+    # Import Resources JSON
+    # resources = {}
+    # with open('sample_files/sample_course_resources.json', 'r') as file:
+    #     resources = json.load(file)
+
+    resources_map = {
+        resource['session_title']: resource['resources']
+        for resource in resources['session_resources']
+    }
+    print("Resources Map is: \n", resources_map)
+    # print("Sample is: ", resources_map.get('Overview of ML Concepts, History, and Applications'));
+    # Generate sessions with their corresponding resources
+    all_sessions = []
+    current_date = start_date
+    
+    for module in course_plan_json['modules']:
+        for sub_module in module['sub_modules']:
+            for topic in sub_module['topics']:
+                session = create_session(
+                    title=topic['title'],
+                    date=current_date,
+                    module_name=module['module_title'],
+                    resources=resources_map.get(topic['title'], {})
+                )
+                all_sessions.append(session)
+                current_date += timedelta(days=3.5)
+    
+    print("All Sessions are: \n", all_sessions)
+
+def get_new_course_id():
+    """Generate a new course ID by incrementing the last course ID"""
+    last_course = courses_collection.find_one(sort=[("course_id", -1)])
+    if last_course:
+        last_course_id = int(last_course["course_id"][2:])
+        new_course_id = f"CS{last_course_id + 1}"
+    else:
+        new_course_id = "CS101"
+    return new_course_id
+
+# if __name__ == "__main__":
+#     course_name = "Introduction to Machine Learning"
+#     start_date = datetime(2022, 9, 1)
+#     duration_weeks = 4
 #     create_course(course_name, start_date, duration_weeks, 3)

create_course3.py

@@ -0,0 +1,609 @@
+from datetime import datetime, timedelta
+import os
+from typing import Dict, List, Any
+from pymongo import MongoClient
+import requests
+import uuid
+import openai
+from openai import OpenAI
+import streamlit as st
+from bson import ObjectId
+from dotenv import load_dotenv
+import json
+import google.generativeai as genai
+from mistralai import Mistral
+
+load_dotenv()
+MONGODB_URI = os.getenv("MONGODB_URI")
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_KEY")
+GEMINI_API_KEY = os.getenv("GEMINI_KEY")
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+
+client = MongoClient(MONGODB_URI)
+db = client['novascholar_db']
+courses_collection = db['courses']
+
+genai.configure(api_key=GEMINI_API_KEY)
+model = genai.GenerativeModel("gemini-1.5-flash")
+
+
+def generate_course_outcomes(api_key, course_name, duration_weeks, sessions_per_week):
+    prompt = f"""
+    You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to generate a comprehensive, academically rigorous set of Course Learning Outcomes (CLOs) for the course {course_name}. These CLOs will serve as a foundation for instructional design and assessment planning.
+
+    Please generate a detailed list of CLOs in JSON format following these specifications:
+
+    1. The CLOs should be clear, concise, and aligned with Bloom's Taxonomy, progressively covering lower-order to higher-order cognitive skills.
+    2. Each CLO must explicitly define the skills, knowledge, or abilities the student is expected to acquire upon completing the course.
+    3. The CLOs must align with the overall course objective and encompass foundational to advanced concepts.
+    4. Use academic language appropriate for higher education or professional training.
+    5. Ensure the CLOs are measurable and actionable.
+    6. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+
+    The JSON response should follow this structure:
+    {{
+        "course_title": "string",
+        "course_description": "string",
+        "learning_outcomes": [
+            {{
+                "outcome_number": CO + number,
+                "outcome_description": "string",
+                "aligned_blooms_taxonomy_level": "string"
+            }}
+        ]
+    }}
+
+    Ensure that:
+    1. Each outcome has a unique outcome_number starting from 1.
+    2. The aligned_blooms_taxonomy_level must be one of the following: "Remember", "Understand", "Apply", "Analyze", "Evaluate", or "Create".
+    3. The total number of CLOs should appropriately cover the breadth and depth of the course content, typically 5-7 CLOs.
+    4. **This Instruction is Strictly followed: DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+
+    """
+    
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are an expert educational AI assistant specializing in course design and curriculum planning. "
+                "Your task is to generate accurate, detailed, and structured educational content that precisely fits "
+                "the specified requirements."
+            ),
+        },
+        {
+            "role": "user",
+            "content": prompt
+        },
+    ]
+    response = model.generate_content(
+        prompt,
+        generation_config=genai.GenerationConfig(
+            response_mime_type="application/json"
+        )
+    )
+    try: 
+        response_json = json.loads(response.text)
+        return response.text;
+    except json.JSONDecodeError as e:
+        print("Error decoding COs JSON response:", e)
+
+def generate_module_outcomes(course_name, course_outcomes, duration_weeks, sessions_per_week):
+    total_sessions = duration_weeks * sessions_per_week
+    prompt = f"""
+        You are an expert educational AI assistant specializing in curriculum design and instructional planning. Your task is to break down the provided Course Learning Outcomes (CLOs) for the course {course_name} into logically structured modules and corresponding Module Learning Outcomes (MLOs). The structure must fit exactly within {duration_weeks} weeks with {total_sessions} total sessions ({sessions_per_week} sessions per week). Each module will be designed to align with specific CLOs and distribute content evenly across the available sessions.
+        
+        Here are the Course Learning Outcomes (CLOs) for the course {course_name} in JSON format:
+        {course_outcomes}
+
+        Please generate the module structure in JSON format following these specifications:
+
+        1. Break the CLOs into logically grouped **modules**, ensuring that each module has a clear focus and progresses from foundational to advanced concepts.
+        2. Each module must include:
+            - A **module title** summarizing its focus.
+            - A list of aligned CLOs that are covered within the module.
+            - Module Learning Outcomes (MLOs) that are measurable and actionable, aligned with the CLOs.
+            - The number of sessions allocated to the module (module_duration_sessions), such that the total sessions across all modules sum up to {total_sessions}.
+        3. Ensure that the module_duration_sessions are evenly distributed while allowing for some variation based on the complexity of the module.
+        4. Progressively distribute content so that earlier modules cover foundational concepts, and later modules cover advanced topics.
+        5. The number of sessions allocated to each module must reflect the relative depth and complexity of its content.
+        6. Ensure all modules fit within {duration_weeks} weeks and {sessions_per_week} sessions per week.
+
+        The JSON response should follow this structure:
+        {{
+            "course_title": "string",
+            "course_description": "string",
+            "total_duration_weeks": {duration_weeks},
+            "sessions_per_week": {sessions_per_week},
+            "total_sessions": {total_sessions},
+            "modules": [
+                {{
+                    "module_title": "string",
+                    "module_duration_sessions": number,
+                    "aligned_CLOs": ["CLO1", "CLO2", ...],
+                    "module_learning_outcomes": [
+                        {{
+                            "outcome_number": "MLO + number",
+                            "outcome_description": "string",
+                            "aligned_blooms_taxonomy_level": "string"
+                        }}
+                    ]
+                }}
+            ]
+        }}
+
+        Ensure that:
+        1. The sum of all module_duration_sessions equals exactly {total_sessions}.
+        2. Each MLO is aligned with its respective CLOs and measurable within the allocated sessions.
+        3. Modules are well-distributed and follow a logical progression of topics.
+        4. **This Instruction is Strictly followed: DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+    """
+    response = model.generate_content(
+        prompt,
+        generation_config=genai.GenerationConfig(
+            response_mime_type="application/json"
+        )
+    )
+    try:
+        response_json = json.loads(response.text)
+        return response.text
+    except json.JSONDecodeError as e:
+        print("Error decoding Modules JSON response:", e)
+
+def generate_submodule_outcomes(course_name, course_outcomes, module_outcomes, duration_weeks, sessions_per_week):
+    prompt = f"""
+        You are an expert educational AI assistant specializing in instructional design. Your task is to further break down each module from the given course structure into submodules. Each submodule will cover specific concepts or topics within the module, along with corresponding Submodule Learning Outcomes (SMLOs).
+
+        Here are the Course Learning Outcomes (CLOs) for the course {course_name}:
+        {course_outcomes}
+        and the Module Learning Outcomes (MLOs) for each module:
+        {module_outcomes}
+        
+        Please follow these guidelines for creating submodules and SMLOs:
+
+        1. For each module, create 2-3 submodules depending on its scope and duration.
+        2. Assign each submodule a clear, concise title summarizing its focus.
+        3. Each submodule must align with at least one Module Learning Outcome (MLO) and, by extension, its parent CLO(s).
+        4. For each submodule, define 1-2 Submodule Learning Outcomes (SMLOs) that are measurable, actionable, and aligned with Bloom's Taxonomy.
+        5. Distribute the total allocated sessions (module_duration_sessions) evenly among submodules, allowing slight variations for complex topics.
+        6. Ensure that submodules progress logically within the module, starting with foundational concepts and advancing to more complex topics.
+        7. Align the submodules with the total sessions allocated to the module to ensure they fit within the course timeline.
+
+        The JSON response should follow this structure:
+        {{
+            "module_title": "string",
+            "submodules": [
+                {{
+                    "submodule_title": "string",
+                    "submodule_duration_sessions": number,
+                    "aligned_MLOs": ["MLO1", "MLO2", ...],
+                    "submodule_learning_outcomes": [
+                        {{
+                            "outcome_number": "SMLO + number",
+                            "outcome_description": "string",
+                            "aligned_blooms_taxonomy_level": "string"
+                        }}
+                    ]
+                }}
+            ]
+        }}
+
+        Ensure that:
+        1. The sum of all submodule_duration_sessions within a module equals the module's allocated sessions (module_duration_sessions).
+        2. SMLOs are specific, measurable, and actionable, aligning with their respective MLOs and CLOs.
+        3. Submodules are logically ordered, with earlier submodules focusing on foundational concepts and later ones covering advanced topics.
+        4. **This Instruction is Strictly followed: DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+    """
+    response = model.generate_content(
+        prompt,
+        generation_config=genai.GenerationConfig(
+            response_mime_type="application/json"
+        )
+    )
+    try:
+        parse_model_response(response.text)
+        return response.text
+    except json.JSONDecodeError as e:
+        print("Error decoding Submodules JSON response:", e)
+
+import json
+import ast
+import re
+import time
+def parse_model_response(response_text):
+    """Enhanced parser for model responses with better error handling.
+    
+    Args:
+        response_text (str): Raw response text from the model
+        
+    Returns:
+        dict or list: Parsed response object
+        
+    Raises:
+        ValueError: If parsing fails
+    """
+    
+    # Remove markdown formatting and whitespace
+    cleaned_text = re.sub(r'```[a-zA-Z]*\n', '', response_text)
+    cleaned_text = cleaned_text.replace('```', '').strip()
+    
+    # Try multiple parsing methods
+    parsing_methods = [
+        # Method 1: Direct JSON parsing
+        lambda x: json.loads(x),
+        
+        # Method 2: AST literal evaluation
+        lambda x: ast.literal_eval(x),
+        
+        # Method 3: Extract and parse content between curly braces
+        lambda x: json.loads(re.search(r'\{.*\}', x, re.DOTALL).group()),
+        
+        # Method 4: Extract and parse content between square brackets
+        lambda x: json.loads(re.search(r'\[.*\]', x, re.DOTALL).group()),
+        
+        # Method 5: Try to fix common JSON formatting issues and parse
+        lambda x: json.loads(x.replace("'", '"').replace('\n', '\\n'))
+    ]
+    
+    last_error = None
+    for parse_method in parsing_methods:
+        try:
+            result = parse_method(cleaned_text)
+            if result:  # Ensure we have actual content
+                return result
+        except Exception as e:
+            last_error = e
+            continue
+            
+    raise ValueError(f"Could not parse the model's response: {last_error}")
+
+def extract_session_titles_concepts(session_data):
+        """Extracts session titles and key concepts from the session data.
+        
+        Args:
+            session_data (dict): Parsed JSON data containing session information
+            
+        Returns:
+            list: List of dictionaries with session titles and key concepts
+        """
+        session_info = []
+        for module in session_data.get("submodules", []):
+            for session in module.get("sessions", []):
+                session_info.append({
+                    "title": session.get("session_title", ""),
+                    "key_concepts": session.get("key_concepts", [])
+                })
+        return session_info
+
+def generate_session_outcomes(course_name, course_outcomes, module_outcomes, submodule_outcomes, duration_weeks, sessions_per_week):
+    prompt = f"""
+        You are an expert educational AI assistant specializing in instructional design and assessment. Your task is to create highly focused and measurable Session-Level Learning Outcomes (SLOs) that are aligned with their parent Submodule Learning Outcomes (SMLOs) and ready to serve as the foundation for rubric-based evaluations.
+
+        ### Context:
+        Course Name: {course_name}
+        Course Outcomes (CLOs): {course_outcomes}
+        Module Outcomes (MLOs): {module_outcomes}
+        Submodule Outcomes (SMLOs): {submodule_outcomes}
+
+        ### Instructions for SLO Generation:
+        1. **For Each Submodule**: Break down its allocated sessions (submodule_duration_sessions) into Session-Level Learning Outcomes (SLOs) that:
+        - Are immediately actionable, measurable, and achievable within a single session (60-90 minutes).
+        - Are directly aligned with their parent SMLO, contributing to its achievement.
+        - Include observable behaviors or outputs using **action-oriented verbs** from Bloom's Taxonomy (e.g., analyze, demonstrate, create, justify).
+        - Are specific and detailed enough to support rubric development.
+
+        2. **Structure for Each Session**:
+        - **Session Title**: A concise and clear session title that captures its focus.
+        - **Prerequisites**: Any prerequisite knowledge or skills required.
+        - **Key Concepts**: Specific concepts or skills that will be covered.
+        - **Session Learning Outcomes (SLOs)**: Include 2-3 outcomes that:
+            - Define precise tasks or objectives for the session.
+            - Specify the expected level of understanding, skill, or performance.
+            - Directly support the parent SMLO while promoting progressive learning.
+
+        3. **Progressive Learning**: Ensure that earlier sessions address foundational knowledge, while later sessions build on this foundation, leading to higher-order skills and integration of concepts.
+
+        4. **Output Format**: Ensure the output follows this strict JSON structure:
+        
+        "submodules": [
+            {{
+                "submodule_title": "string",
+                "sessions": [
+                    {{
+                        "session_number": number,
+                        "session_title": "string",
+                        "prerequisites": ["string"],
+                        "key_concepts": ["string"],
+                        "session_learning_outcomes": [
+                            {{
+                                "outcome_number": "SLO + number",
+                                "outcome_description": "string",
+                                "aligned_smlo": "SMLO + number",
+                                "bloom_taxonomy_level": "string"
+                            }}
+                        ]
+                    }}
+                ]
+            }}
+        ]
+
+        ### Example of Rubric-Ready SLOs:
+        - **Poor Example**: "Understand agile methodologies."
+        - **Good Example**: "Identify and describe the key principles of the Agile Manifesto, providing examples of how each principle applies to software development."
+
+        **Important Instructions**:
+        1. Ensure every SLO can be directly translated into rubric criteria (e.g., clarity, accuracy, application).
+        2. **DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT USE BACKTICKS (```), AND DO NOT INCLUDE ANY OTHER TEXT EXCEPT THE JSON RESPONSE. START WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.**
+        3. Make sure every SLO is relevant to its parent SMLO.
+    """
+    
+    response = model.generate_content(
+        prompt,
+        generation_config=genai.GenerationConfig(
+            response_mime_type="application/json"
+        )
+    )
+    try:
+        parse_model_response(response.text)
+        return response.text
+    except json.JSONDecodeError as e:
+        print("Error decoding Session Learning Outcomes JSON response:", e)
+
+
+def merge_course_structure(cos, module_outcomes, submodules_los, sample_sessions_se):
+    # Load JSON data
+    cos_data = cos
+    module_outcomes_data = module_outcomes
+    submodules_los_data = submodules_los
+    sample_sessions_se_data = sample_sessions_se
+
+    # Create a mapping of submodule titles to their sessions
+    submodule_sessions_map = {}
+    for submodule in sample_sessions_se_data['submodules']:
+        submodule_sessions_map[submodule['submodule_title']] = submodule['sessions']
+
+    # Create a mapping of module titles to their submodules
+    module_submodules_map = {}
+    for module in submodules_los_data['modules']:
+        module_submodules_map[module['module_title']] = module['submodules']
+
+    # Merge submodules into modules
+    for module in module_outcomes_data['modules']:
+        module_title = module['module_title']
+        if module_title in module_submodules_map:
+            submodules = module_submodules_map[module_title]
+            for submodule in submodules:
+                submodule_title = submodule['submodule_title']
+                if submodule_title in submodule_sessions_map:
+                    submodule['sessions'] = submodule_sessions_map[submodule_title]
+            module['submodules'] = submodules
+
+    # Merge modules into course structure
+    course_structure = cos_data
+    course_structure['modules'] = module_outcomes_data['modules']
+
+    return course_structure
+
+def generate_session_resources(api_key, course_title, session_titles: List[str]):
+    """
+    Generate relevant resources for each session title separately
+    """
+    resources_prompt = f"""
+        You are an expert educational content curator with deep knowledge of instructional design and high-quality resource selection. Your task is to provide session-specific learning resources and course-level reference books for the course: {course_title}.
+
+        Guidelines for Resource Curation:
+        1. For each session, suggest **highly relevant and accurate learning resources** based on the session title and key concepts provided.
+        2. For the course as a whole, provide at most two **top reference books** that comprehensively cover the course objectives, including both academic and practical perspectives.
+        3. Resources can include:
+            - **Web articles or blogs** (ensure they are from authoritative and credible sources)
+            - **Videos** (e.g., YouTube or other educational platforms)
+            - **PDFs, PPTs, or other downloadable formats**
+            - **Official documentation** for tools, platforms, or technologies
+        4. Provide **multiple resources per session**, tailored to the topic's depth and complexity. Collectively, the number should not exceed 3.
+        5. **IMPORTANT: MAKE SURE READINGS AND VIDEOS ARE GIVEN SEPARATELY. READINGS SHOULD NOT CONTAIN VIDEOS, IT SHOULD ONLY CONTAIN READING MATERIAL AND VICE-VERSA FOR VIDEOS**
+        6. Ensure all URLs are **active and accessible**. Resources must be up-to-date, and links should work reliably.
+        7. Reference books for the course should be **real, recently published works** and relevant to the course-level outcomes.
+
+        Output Format:
+        {{
+            "course_reference_books": [
+                {{
+                    "title": "string",
+                    "author": "string",
+                    "publisher": "string",
+                    "year": number,
+                    "description": "string"
+                }}
+            ],
+            "session_resources": [
+                {{
+                    "session_title": "string",
+                    "resources": {{
+                        "readings": [
+                            {{
+                                "title": "string",
+                                "url": "string",
+                                "type": "string",
+                                "estimated_read_time": "string"
+                            }}
+                        ],
+                        "videos": [
+                            {{
+                                "title": "string",
+                                "url": "string",
+                                "type": "string",
+                                "duration": "string"
+                            }}
+                        ]
+                    }}
+                }}
+            ]
+        }}
+
+        Additional Instructions:
+        - Ensure **property names are enclosed in double quotes (")** and values are properly formatted.
+        - Reference books should include **a brief description** to explain why they are relevant to the course.
+        - Responses should be concise, structured, and focused exclusively on the requested information.
+        - ***IMPORTANT: DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.***
+
+        Here are the session titles and key concepts for which you need to generate resources: {session_titles}.
+    """
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are an expert educational content curator, focused on providing accurate and relevant learning resources.",
+        },
+        {
+            "role": "user",
+            "content": resources_prompt
+        },
+    ]
+
+    try:
+        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
+        response = client.chat.completions.create(
+            model="llama-3.1-sonar-small-128k-online",
+            messages=messages
+        )
+        print("Response is: \n", response.choices[0].message.content)
+        # try:
+        #     return json.loads(response.choices[0].message.content)
+        # except json.JSONDecodeError as e:
+        #     st.error(f"Failed to decode JSON response: {e}")
+        #     return None
+        return response.choices[0].message.content
+    except Exception as e:
+        st.error(f"Failed to generate resources: {e}")
+        return None
+
+def generate_resources_by_titles_chunking(session_titles, course_title):
+    def chunk_list(lst, chunk_size):
+        for i in range(0, len(lst), chunk_size):
+            yield lst[i:i + chunk_size]
+    
+    all_session_resources = []
+    course_reference_books = None  # Initialize this variable
+
+    # Process each chunk of session titles
+    for i, chunk in enumerate(chunk_list(session_titles, 10)):
+        session_resources_chunk = generate_session_resources(PERPLEXITY_API_KEY, course_title, chunk)
+        if session_resources_chunk:
+            if "session_resources" in session_resources_chunk:
+                # Parse the JSON string if it's a string
+                if isinstance(session_resources_chunk, str):
+                    session_resources_chunk = json.loads(session_resources_chunk)
+                all_session_resources.extend(session_resources_chunk["session_resources"])
+            # all_session_resources.extend(session_resources_chunk["session_resources"])
+            # Only take the course_reference_books from the first chunk
+            else:
+                print("Some problem occured. Session resources chunk:", session_resources_chunk)
+            
+            if i == 0 and "course_reference_books" in session_resources_chunk:
+                course_reference_books = session_resources_chunk.get("course_reference_books", [])
+        time.sleep(2)
+
+    # Combine all session resources into a single dictionary
+    complete_session_resources = {
+        "course_reference_books": course_reference_books,
+        "session_resources": all_session_resources
+    }
+    # Save the complete session resources to a JSON file
+    output_file_path = 'sample_files/session_resources2.json'
+    with open(output_file_path, 'w') as outfile:
+        try:
+            json.dump(complete_session_resources, outfile, indent=4)
+        except Exception as e:
+            print(f"Failed to save session resources to file: {e}")
+    
+     # Debug print before return
+    print("Type of complete_session_resources:", type(complete_session_resources))
+    print("Content of complete_session_resources:", complete_session_resources)
+    
+
+    return complete_session_resources
+
+if __name__ == "__main__":
+    # course_name = "Introduction to Machine Learning"
+    # duration_weeks = 12
+    # sessions_per_week = 2
+    # Load COs from JSON file:
+    # with open('sample_files/cos.json', 'r') as file:
+    #     course_outcomes = json.load(file)
+    
+    # # Load MLOs from JSON file:
+    # with open('sample_files/module_outcomes.json', 'r') as file:
+    #     module_outcomes = json.load(file)
+    
+    # # Load SMLOs from JSON file:
+    # with open('sample_files/submodules_los.json', 'r') as file:
+    #     submodules_outcomes = json.load(file)
+
+    # print("Generating Course Outcomes...")
+    # course_outcomes = generate_course_outcomes(GEMINI_API_KEY, course_name, duration_weeks, sessions_per_week)
+    # print("Generating Modules...")
+    # module_outcomes = generate_module_outcomes(course_name, course_outcomes, duration_weeks, sessions_per_week)
+    # print("Generating Submodules...")
+    # submodules_outcomes = generate_submodule_outcomes(course_name, course_outcomes, module_outcomes, duration_weeks, sessions_per_week)
+    # print("Generating Sessions...")
+    # session_outcomes = generate_session_outcomes(course_name, course_outcomes, module_outcomes, submodules_outcomes, duration_weeks, sessions_per_week)
+    # print(session_outcomes)
+    # print("Extracting Session Titles...")
+    # # Load Sessions from JSON file:
+    with open('sample_files/sample_sessions_se.json', 'r') as file:
+        session_data = json.load(file)
+    session_titles_concepts = extract_session_titles_concepts(session_data)
+    print(session_titles_concepts)
+    # print("Generating Session Resources...")
+    # # Chunk the session titles into batches of 10
+    # def chunk_list(lst, chunk_size):
+    #     for i in range(0, len(lst), chunk_size):
+    #         yield lst[i:i + chunk_size]
+
+    # Extract session titles from session_titles_concepts
+    session_titles = [session["title"] for session in session_titles_concepts]
+
+    # # Initialize an empty list to store all session resources
+    # all_session_resources = []
+
+    # # Process each chunk of session titles
+    # for chunk in chunk_list(session_titles, 10):
+    #     session_resources_chunk = generate_session_resources(PERPLEXITY_API_KEY, course_name, chunk)
+    #     if session_resources_chunk:
+    #         all_session_resources.extend(json.loads(session_resources_chunk)["session_resources"])
+    #     time.sleep(2)
+
+    # # Combine all session resources into a single dictionary
+    # complete_session_resources = {
+    #     "course_reference_books": json.loads(session_resources_chunk)["course_reference_books"],
+    #     "session_resources": all_session_resources
+    # }
+
+    # # Save the complete session resources to a JSON file
+    # output_file_path = 'sample_files/session_resources.json'
+    # with open(output_file_path, 'w') as outfile:
+    #     json.dump(complete_session_resources, outfile, indent=4)
+    # print(complete_session_resources)
+
+
+    # session_resources = generate_session_resources(PERPLEXITY_API_KEY, course_name, session_titles_concepts)
+    
+    # # Save session resources to a JSON file
+    # output_file_path = 'sample_files/session_resources.json'
+    # with open(output_file_path, 'w') as outfile:
+    #     json.dump(session_resources, outfile, indent=4)
+    # print(session_resources)
+
+    # Create course structure
+    # course_structure = merge_course_structure(course_outcomes, module_outcomes, submodules_outcomes, session_outcomes)
+    # Save course structure to a JSON file
+    # output_file_path = 'sample_files/course_structure2.json'
+    # with open(output_file_path, 'w') as outfile:
+    #     json.dump(course_structure, outfile, indent=4)
+    # print(course_structure)
+    resources = generate_resources_by_titles_chunking(session_titles, "Software Engineering")
+    print(resources)
+    
+
+
+

db.py

@@ -1,696 +1,696 @@
-# Setup for MongoDB
-from pymongo import MongoClient
-from datetime import datetime
-from werkzeug.security import generate_password_hash
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-MONGO_URI = os.getenv("MONGO_URI")
-
-client = MongoClient(MONGO_URI)
-try:
-    client.admin.command("ping")
-    print("MongoDB connection successful")
-except Exception as e:
-    print(f"MongoDB connection failed: {e}")
-
-db = client["novascholar_db"]
-
-########
-# Research Assistant Schema
-research_assistant_schema = {
-    "bsonType": "object",
-    "required": ["full_name", "password", "email", "courses_assisted"],
-    "properties": {
-        "full_name": {
-            "bsonType": "string",
-            "description": "Full name of the research assistant",
-        },
-        "password": {
-            "bsonType": "string",
-            "description": "Hashed password of the research assistant",
-        },
-        "email": {
-            "bsonType": "string",
-            "description": "Email address of the research assistant",
-        },
-        "courses_assisted": {
-            "bsonType": "array",
-            "description": "List of courses the research assistant is assisting",
-            "items": {
-                "bsonType": "object",
-                "required": ["course_id"],
-                "properties": {
-                    "course_id": {
-                        "bsonType": "string",
-                        "description": "ID of the course",
-                    }
-                },
-            },
-        },
-    },
-}
-
-# Create research assistants collection
-research_assistants_collection = db["research_assistants"]
-
-# Create indexes
-research_assistants_collection.create_index("full_name", unique=True)
-research_assistants_collection.create_index("email", unique=True)
-
-
-# Optional: Sample data insertion function
-def insert_sample_research_assistants():
-    sample_research_assistants = [
-        {
-            "full_name": "John Doe RA",
-            "password": generate_password_hash("password123"),
-            "email": "[email protected]",
-            "courses_assisted": [{"course_id": "CS101"}, {"course_id": "CS102"}],
-        }
-    ]
-
-    try:
-        research_assistants_collection.insert_many(sample_research_assistants)
-        print("Sample research assistants inserted successfully!")
-    except Exception as e:
-        print(f"Error inserting sample research assistants: {e}")
-
-
-###########
-
-###############
-# Add after research assistant schema
-
-# Analyst Schema
-analyst_schema = {
-    "bsonType": "object",
-    "required": ["full_name", "password", "email", "courses_analyzed"],
-    "properties": {
-        "full_name": {"bsonType": "string", "description": "Full name of the analyst"},
-        "password": {
-            "bsonType": "string",
-            "description": "Hashed password of the analyst",
-        },
-        "email": {"bsonType": "string", "description": "Email address of the analyst"},
-        "courses_analyzed": {
-            "bsonType": "array",
-            "description": "List of courses the analyst is analyzing",
-            "items": {
-                "bsonType": "object",
-                "required": ["course_id"],
-                "properties": {
-                    "course_id": {
-                        "bsonType": "string",
-                        "description": "ID of the course",
-                    }
-                },
-            },
-        },
-    },
-}
-
-# Create analysts collection
-analysts_collection = db["analysts"]
-
-# Create indexes for analysts
-analysts_collection.create_index("full_name", unique=True)
-analysts_collection.create_index("email", unique=True)
-
-
-def insert_sample_analysts():
-    sample_analysts = [
-        {
-            "full_name": "jane",
-            "password": generate_password_hash("jane"),
-            "email": "[email protected]",
-            "courses_analyzed": [{"course_id": "CS101"}, {"course_id": "CS102"}],
-        }
-    ]
-
-    try:
-        analysts_collection.insert_many(sample_analysts)
-        print("Sample analysts inserted successfully!")
-    except Exception as e:
-        print(f"Error inserting sample analysts: {e}")
-
-
-##############@
-
-
-# Define the course schema
-course_schema = {
-    "bsonType": "object",
-    "required": [
-        "course_id",
-        "title",
-        "description",
-        "faculty",
-        "faculty_id",
-        "duration",
-        "created_at",
-    ],
-    "properties": {
-        "course_id": {
-            "bsonType": "string",
-            "description": "Unique identifier for the course",
-        },
-        "title": {"bsonType": "string", "description": "Title of the course"},
-        "description": {
-            "bsonType": "string",
-            "description": "Description of the course",
-        },
-        "faculty": {"bsonType": "string", "description": "Name of the faculty"},
-        "duration": {"bsonType": "string", "description": "Duration of the course"},
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the course was created",
-        },
-        "sessions": {
-            "bsonType": "array",
-            "description": "List of sessions associated with the course",
-            "items": {
-                "bsonType": "object",
-                "required": ["session_id", "title", "date", "status", "created_at"],
-                "properties": {
-                    "session_id": {
-                        "bsonType": "string",
-                        "description": "Unique identifier for the session",
-                    },
-                    "title": {
-                        "bsonType": "string",
-                        "description": "Title of the session",
-                    },
-                    "date": {"bsonType": "date", "description": "Date of the session"},
-                    "status": {
-                        "bsonType": "string",
-                        "description": "Status of the session (e.g., completed, upcoming)",
-                    },
-                    "created_at": {
-                        "bsonType": "date",
-                        "description": "Date when the session was created",
-                    },
-                    "pre_class": {
-                        "bsonType": "object",
-                        "description": "Pre-class segment data",
-                        "properties": {
-                            "resources": {
-                                "bsonType": "array",
-                                "description": "List of pre-class resources",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["type", "title", "url"],
-                                    "properties": {
-                                        "type": {
-                                            "bsonType": "string",
-                                            "description": "Type of resource (e.g., pdf, video)",
-                                        },
-                                        "title": {
-                                            "bsonType": "string",
-                                            "description": "Title of the resource",
-                                        },
-                                        "url": {
-                                            "bsonType": "string",
-                                            "description": "URL of the resource",
-                                        },
-                                        "vector": {
-                                            "bsonType": "array",
-                                            "description": "Vector representation of the resource",
-                                            "items": {"bsonType": "double"},
-                                        },
-                                    },
-                                },
-                            },
-                            "completion_required": {
-                                "bsonType": "bool",
-                                "description": "Indicates if completion of pre-class resources is required",
-                            },
-                        },
-                    },
-                    "in_class": {
-                        "bsonType": "object",
-                        "description": "In-class segment data",
-                        "properties": {
-                            "topics": {
-                                "bsonType": "array",
-                                "description": "List of topics covered in the session",
-                                "items": {"bsonType": "string"},
-                            },
-                            "quiz": {
-                                "bsonType": "object",
-                                "description": "Quiz data",
-                                "properties": {
-                                    "title": {
-                                        "bsonType": "string",
-                                        "description": "Title of the quiz",
-                                    },
-                                    "questions": {
-                                        "bsonType": "int",
-                                        "description": "Number of questions in the quiz",
-                                    },
-                                    "duration": {
-                                        "bsonType": "int",
-                                        "description": "Duration of the quiz in minutes",
-                                    },
-                                },
-                            },
-                            "polls": {
-                                "bsonType": "array",
-                                "description": "List of polls conducted during the session",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["question", "options"],
-                                    "properties": {
-                                        "question": {
-                                            "bsonType": "string",
-                                            "description": "Poll question",
-                                        },
-                                        "options": {
-                                            "bsonType": "array",
-                                            "description": "List of poll options",
-                                            "items": {"bsonType": "string"},
-                                        },
-                                        "responses": {
-                                            "bsonType": "object",
-                                            "description": "Responses to the poll",
-                                            "additionalProperties": {"bsonType": "int"},
-                                        },
-                                    },
-                                },
-                            },
-                        },
-                    },
-                    "post_class": {
-                        "bsonType": "object",
-                        "description": "Post-class segment data",
-                        "properties": {
-                            "assignments": {
-                                "bsonType": "array",
-                                "description": "List of assignments",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["id", "title", "due_date", "status"],
-                                    "properties": {
-                                        "id": {
-                                            "bsonType": "int",
-                                            "description": "Assignment ID",
-                                        },
-                                        "title": {
-                                            "bsonType": "string",
-                                            "description": "Title of the assignment",
-                                        },
-                                        "due_date": {
-                                            "bsonType": "date",
-                                            "description": "Due date of the assignment",
-                                        },
-                                        "status": {
-                                            "bsonType": "string",
-                                            "description": "Status of the assignment (e.g., pending, completed)",
-                                        },
-                                        "submissions": {
-                                            "bsonType": "array",
-                                            "description": "List of submissions",
-                                            "items": {
-                                                "bsonType": "object",
-                                                "required": [
-                                                    "student_id",
-                                                    "file_url",
-                                                    "submitted_at",
-                                                ],
-                                                "properties": {
-                                                    "student_id": {
-                                                        "bsonType": "string",
-                                                        "description": "ID of the student who submitted the assignment",
-                                                    },
-                                                    "file_url": {
-                                                        "bsonType": "string",
-                                                        "description": "URL of the submitted file",
-                                                    },
-                                                    "submitted_at": {
-                                                        "bsonType": "date",
-                                                        "description": "Date when the assignment was submitted",
-                                                    },
-                                                },
-                                            },
-                                        },
-                                    },
-                                },
-                            }
-                        },
-                    },
-                },
-            },
-        },
-    },
-}
-
-# Create the collection with the schema
-# db.create_collection("courses_collection2", validator={"$jsonSchema": course_schema})
-
-# sample_course = {
-#     "course_id": "CS101",
-#     "title": "Introduction to Computer Science",
-#     "description": "This course covers the basics of computer science and programming.",
-#     "faculty": "Dr. John Doe",
-#     "faculty_id": "F101",
-#     "duration": "10 weeks",
-#     "created_at": datetime.utcnow(),
-#     "sessions": [
-#         {
-#             "session_id": "S101",
-#             "title": "Introduction to Programming Fundamentals",
-#             "date": datetime.utcnow() - timedelta(days=7),
-#             "status": "completed",
-#             "created_at": datetime.utcnow() - timedelta(days=7),
-#             "pre_class": {
-#                 "resources": [
-#                     {
-#                         "type": "pdf",
-#                         "title": "Introduction to Python Basics",
-#                         "url": "/assets/python_basics.pdf",
-#                         "vector": [0.1, 0.2, 0.3]  # Example vector
-#                     }
-#                 ],
-#                 "completion_required": True
-#             },
-#             "in_class": {
-#                 "topics": ["Variables", "Data Types", "Basic Operations"],
-#                 "quiz": {
-#                     "title": "Python Basics Quiz",
-#                     "questions": 5,
-#                     "duration": 15
-#                 },
-#                 "polls": [
-#                     {
-#                         "question": "How comfortable are you with Python syntax?",
-#                         "options": ["Very", "Somewhat", "Not at all"],
-#                         "responses": {"Very": 10, "Somewhat": 5, "Not at all": 2}
-#                     }
-#                 ]
-#             },
-#             "post_class": {
-#                 "assignments": [
-#                     {
-#                         "id": 1,
-#                         "title": "Basic Python Programs",
-#                         "due_date": datetime.utcnow() + timedelta(days=2),
-#                         "status": "pending",
-#                         "submissions": []
-#                     }
-#                 ]
-#             }
-#         },
-#         {
-#             "session_id": "S102",
-#             "title": "Control Flow and Functions",
-#             "date": datetime.utcnow() - timedelta(days=3),
-#             "status": "completed",
-#             "created_at": datetime.utcnow() - timedelta(days=3),
-#             "pre_class": {
-#                 "resources": [
-#                     {
-#                         "type": "pdf",
-#                         "title": "Control Flow in Python",
-#                         "url": "/assets/control_flow.pdf",
-#                         "vector": [0.4, 0.5, 0.6]  # Example vector
-#                     }
-#                 ],
-#                 "completion_required": True
-#             },
-#             "in_class": {
-#                 "topics": ["If-else statements", "Loops", "Function definitions"],
-#                 "quiz": {
-#                     "title": "Control Flow Quiz",
-#                     "questions": 8,
-#                     "duration": 20
-#                 },
-#                 "polls": [
-#                     {
-#                         "question": "Which loop type do you find more intuitive?",
-#                         "options": ["For loops", "While loops", "Both"],
-#                         "responses": {"For loops": 12, "While loops": 8, "Both": 10}
-#                     }
-#                 ]
-#             },
-#             "post_class": {
-#                 "assignments": [
-#                     {
-#                         "id": 2,
-#                         "title": "Function Implementation Exercise",
-#                         "due_date": datetime.utcnow() + timedelta(days=4),
-#                         "status": "pending",
-#                         "submissions": []
-#                     }
-#                 ]
-#             }
-#         }
-#     ]
-# }
-courses_collection2 = db["courses_collection2"]
-
-
-# Define the users schema
-users_schema = {
-    "bsonType": "object",
-    "required": ["user_id", "username", "password", "role", "created_at"],
-    "properties": {
-        "user_id": {
-            "bsonType": "string",
-            "description": "Unique identifier for the user",
-        },
-        "username": {"bsonType": "string", "description": "Name of the User"},
-        "password": {"bsonType": "string", "description": "Password of the user"},
-        "role": {
-            "bsonType": "string",
-            "description": "Type of user (e.g., student, faculty)",
-        },
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the user was created",
-        },
-    },
-}
-# Create the collection with the schema
-# db.create_collection("users", validator={"$jsonSchema": users_schema})
-users_collection = db["users"]
-
-
-# Defining the Student Collection
-student_schema = {
-    "bsonType": "object",
-    "required": ["SID", "full_name", "password", "enrolled_courses", "created_at"],
-    "properties": {
-        "SID": {
-            "bsonType": "string",
-            "description": "Unique identifier for the student",
-        },
-        "full_name": {"bsonType": "string", "description": "Full name of the student"},
-        "password": {
-            "bsonType": "string",
-            "description": "Hashed password of the student",
-        },
-        "enrolled_courses": {
-            "bsonType": "array",
-            "description": "List of courses the student is enrolled in",
-            "items": {
-                "bsonType": "object",
-                "required": ["course_id", "title"],
-                "properties": {
-                    "course_id": {
-                        "bsonType": "string",
-                        "description": "Unique identifier for the course",
-                    },
-                    "title": {
-                        "bsonType": "string",
-                        "description": "Title of the course",
-                    },
-                },
-            },
-        },
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the student was created",
-        },
-    },
-}
-# Defining the Faculty Collection
-faculty_schema = {
-    "bsonType": "object",
-    "required": ["TID", "full_name", "password", "courses_taught", "created_at"],
-    "properties": {
-        "TID": {
-            "bsonType": "string",
-            "description": "Unique identifier for the faculty",
-        },
-        "full_name": {"bsonType": "string", "description": "Full name of the faculty"},
-        "password": {
-            "bsonType": "string",
-            "description": "Hashed password of the faculty",
-        },
-        "courses_taught": {
-            "bsonType": "array",
-            "description": "List of courses the faculty is teaching",
-            "items": {
-                "bsonType": "object",
-                "required": ["course_id", "title"],
-                "properties": {
-                    "course_id": {
-                        "bsonType": "string",
-                        "description": "Unique identifier for the course",
-                    },
-                    "title": {
-                        "bsonType": "string",
-                        "description": "Title of the course",
-                    },
-                },
-            },
-        },
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the faculty was created",
-        },
-    },
-}
-# Creating the Collections
-# db.create_collection("students", validator={"$jsonSchema": student_schema})
-# db.create_collection("faculty", validator={"$jsonSchema": faculty_schema})
-
-students_collection = db["students"]
-faculty_collection = db["faculty"]
-
-# Defining the Vector Collection Schema
-vector_schema = {
-    "bsonType": "object",
-    "required": ["resource_id", "vector"],
-    "properties": {
-        "resource_id": {
-            "bsonType": "objectId",
-            "description": "Unique identifier for the resource",
-        },
-        "vector": {
-            "bsonType": "array",
-            "description": "Vector representation of the resource",
-            "items": {"bsonType": "double"},
-        },
-        "text": {"bsonType": "string", "description": "Text content of the resource"},
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the vector was created",
-        },
-    },
-}
-# Creating the Vector Collection
-# db.create_collection("vectors", validator={"$jsonSchema": vector_schema})
-vectors_collection = db["vectors"]
-
-
-# Creating a Chat-History Collection
-# Creating a Chat-History Collection
-chat_history_schema = {
-    "bsonType": "object",
-    "required": ["user_id", "session_id", "messages", "timestamp"],
-    "properties": {
-        "user_id": {
-            "bsonType": "objectId",
-            "description": "Unique identifier for the user",
-        },
-        "session_id": {
-            "bsonType": "string",
-            "description": "Identifier for the session",
-        },
-        "timestamp": {
-            "bsonType": "date",
-            "description": "Timestamp when the chat session started",
-        },
-        "messages": {
-            "bsonType": "array",
-            "description": "List of chat messages",
-            "items": {
-                "bsonType": "object",
-                "properties": {
-                    "prompt": {
-                        "bsonType": "string",
-                        "description": "User's question or prompt",
-                    },
-                    "response": {
-                        "bsonType": "string",
-                        "description": "Assistant's response",
-                    },
-                    "timestamp": {
-                        "bsonType": "date",
-                        "description": "Timestamp of the message",
-                    },
-                },
-            },
-        },
-    },
-}
-
-# Create the collection with the schema
-# db.create_collection("chat_history", validator={"$jsonSchema": chat_history_schema})
-chat_history_collection = db["chat_history"]
-
-
-# Database setup for Research Assistant
-# Research Assistant Schema
-research_assistant_schema = {
-    "bsonType": "object",
-    "required": ["full_name", "password", "email", "courses_assisted"],
-    "properties": {
-        "full_name": {
-            "bsonType": "string",
-            "description": "Full name of the research assistant",
-        },
-        "password": {
-            "bsonType": "string",
-            "description": "Hashed password of the research assistant",
-        },
-        "email": {
-            "bsonType": "string",
-            "description": "Email address of the research assistant",
-        },
-        "courses_assisted": {
-            "bsonType": "array",
-            "description": "List of courses the research assistant is assisting",
-            "items": {
-                "bsonType": "object",
-                "required": ["course_id"],
-                "properties": {
-                    "course_id": {
-                        "bsonType": "string",
-                        "description": "ID of the course",
-                    }
-                },
-            },
-        },
-    },
-}
-
-# Create research assistants collection
-research_assistants_collection = db["research_assistants"]
-
-# Create indexes
-research_assistants_collection.create_index("full_name", unique=True)
-research_assistants_collection.create_index("email", unique=True)
-
-
-# Optional: Sample data insertion function
-# def insert_sample_research_assistants():
-#     sample_research_assistants = [
-#         {
-#             "full_name": "John Doe RA",
-#             "password": generate_password_hash("password123"),
-#             "email": "[email protected]",
-#             "courses_assisted": [{"course_id": "CS101"}, {"course_id": "CS102"}],
-#         }
-#     ]
-
-#     try:
-#         research_assistants_collection.insert_many(sample_research_assistants)
-#         print("Sample research assistants inserted successfully!")
-#     except Exception as e:
-#         print(f"Error inserting sample research assistants: {e}")
-
-# if __name__ == "__main__":
-#     insert_sample_analysts()
+# Setup for MongoDB
+from pymongo import MongoClient
+from datetime import datetime
+from werkzeug.security import generate_password_hash
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+MONGO_URI = os.getenv("MONGO_URI")
+
+client = MongoClient(MONGO_URI)
+try:
+    client.admin.command("ping")
+    print("MongoDB connection successful")
+except Exception as e:
+    print(f"MongoDB connection failed: {e}")
+
+db = client["novascholar_db"]
+
+########
+# Research Assistant Schema
+research_assistant_schema = {
+    "bsonType": "object",
+    "required": ["full_name", "password", "email", "courses_assisted"],
+    "properties": {
+        "full_name": {
+            "bsonType": "string",
+            "description": "Full name of the research assistant",
+        },
+        "password": {
+            "bsonType": "string",
+            "description": "Hashed password of the research assistant",
+        },
+        "email": {
+            "bsonType": "string",
+            "description": "Email address of the research assistant",
+        },
+        "courses_assisted": {
+            "bsonType": "array",
+            "description": "List of courses the research assistant is assisting",
+            "items": {
+                "bsonType": "object",
+                "required": ["course_id"],
+                "properties": {
+                    "course_id": {
+                        "bsonType": "string",
+                        "description": "ID of the course",
+                    }
+                },
+            },
+        },
+    },
+}
+
+# Create research assistants collection
+research_assistants_collection = db["research_assistants"]
+
+# Create indexes
+research_assistants_collection.create_index("full_name", unique=True)
+research_assistants_collection.create_index("email", unique=True)
+
+
+# Optional: Sample data insertion function
+def insert_sample_research_assistants():
+    sample_research_assistants = [
+        {
+            "full_name": "John Doe RA",
+            "password": generate_password_hash("password123"),
+            "email": "[email protected]",
+            "courses_assisted": [{"course_id": "CS101"}, {"course_id": "CS102"}],
+        }
+    ]
+
+    try:
+        research_assistants_collection.insert_many(sample_research_assistants)
+        print("Sample research assistants inserted successfully!")
+    except Exception as e:
+        print(f"Error inserting sample research assistants: {e}")
+
+
+###########
+
+###############
+# Add after research assistant schema
+
+# Analyst Schema
+analyst_schema = {
+    "bsonType": "object",
+    "required": ["full_name", "password", "email", "courses_analyzed"],
+    "properties": {
+        "full_name": {"bsonType": "string", "description": "Full name of the analyst"},
+        "password": {
+            "bsonType": "string",
+            "description": "Hashed password of the analyst",
+        },
+        "email": {"bsonType": "string", "description": "Email address of the analyst"},
+        "courses_analyzed": {
+            "bsonType": "array",
+            "description": "List of courses the analyst is analyzing",
+            "items": {
+                "bsonType": "object",
+                "required": ["course_id"],
+                "properties": {
+                    "course_id": {
+                        "bsonType": "string",
+                        "description": "ID of the course",
+                    }
+                },
+            },
+        },
+    },
+}
+
+# Create analysts collection
+analysts_collection = db["analysts"]
+
+# Create indexes for analysts
+analysts_collection.create_index("full_name", unique=True)
+analysts_collection.create_index("email", unique=True)
+
+
+def insert_sample_analysts():
+    sample_analysts = [
+        {
+            "full_name": "jane",
+            "password": generate_password_hash("jane"),
+            "email": "[email protected]",
+            "courses_analyzed": [{"course_id": "CS101"}, {"course_id": "CS102"}],
+        }
+    ]
+
+    try:
+        analysts_collection.insert_many(sample_analysts)
+        print("Sample analysts inserted successfully!")
+    except Exception as e:
+        print(f"Error inserting sample analysts: {e}")
+
+
+##############@
+
+
+# Define the course schema
+course_schema = {
+    "bsonType": "object",
+    "required": [
+        "course_id",
+        "title",
+        "description",
+        "faculty",
+        "faculty_id",
+        "duration",
+        "created_at",
+    ],
+    "properties": {
+        "course_id": {
+            "bsonType": "string",
+            "description": "Unique identifier for the course",
+        },
+        "title": {"bsonType": "string", "description": "Title of the course"},
+        "description": {
+            "bsonType": "string",
+            "description": "Description of the course",
+        },
+        "faculty": {"bsonType": "string", "description": "Name of the faculty"},
+        "duration": {"bsonType": "string", "description": "Duration of the course"},
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the course was created",
+        },
+        "sessions": {
+            "bsonType": "array",
+            "description": "List of sessions associated with the course",
+            "items": {
+                "bsonType": "object",
+                "required": ["session_id", "title", "date", "status", "created_at"],
+                "properties": {
+                    "session_id": {
+                        "bsonType": "string",
+                        "description": "Unique identifier for the session",
+                    },
+                    "title": {
+                        "bsonType": "string",
+                        "description": "Title of the session",
+                    },
+                    "date": {"bsonType": "date", "description": "Date of the session"},
+                    "status": {
+                        "bsonType": "string",
+                        "description": "Status of the session (e.g., completed, upcoming)",
+                    },
+                    "created_at": {
+                        "bsonType": "date",
+                        "description": "Date when the session was created",
+                    },
+                    "pre_class": {
+                        "bsonType": "object",
+                        "description": "Pre-class segment data",
+                        "properties": {
+                            "resources": {
+                                "bsonType": "array",
+                                "description": "List of pre-class resources",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["type", "title", "url"],
+                                    "properties": {
+                                        "type": {
+                                            "bsonType": "string",
+                                            "description": "Type of resource (e.g., pdf, video)",
+                                        },
+                                        "title": {
+                                            "bsonType": "string",
+                                            "description": "Title of the resource",
+                                        },
+                                        "url": {
+                                            "bsonType": "string",
+                                            "description": "URL of the resource",
+                                        },
+                                        "vector": {
+                                            "bsonType": "array",
+                                            "description": "Vector representation of the resource",
+                                            "items": {"bsonType": "double"},
+                                        },
+                                    },
+                                },
+                            },
+                            "completion_required": {
+                                "bsonType": "bool",
+                                "description": "Indicates if completion of pre-class resources is required",
+                            },
+                        },
+                    },
+                    "in_class": {
+                        "bsonType": "object",
+                        "description": "In-class segment data",
+                        "properties": {
+                            "topics": {
+                                "bsonType": "array",
+                                "description": "List of topics covered in the session",
+                                "items": {"bsonType": "string"},
+                            },
+                            "quiz": {
+                                "bsonType": "object",
+                                "description": "Quiz data",
+                                "properties": {
+                                    "title": {
+                                        "bsonType": "string",
+                                        "description": "Title of the quiz",
+                                    },
+                                    "questions": {
+                                        "bsonType": "int",
+                                        "description": "Number of questions in the quiz",
+                                    },
+                                    "duration": {
+                                        "bsonType": "int",
+                                        "description": "Duration of the quiz in minutes",
+                                    },
+                                },
+                            },
+                            "polls": {
+                                "bsonType": "array",
+                                "description": "List of polls conducted during the session",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["question", "options"],
+                                    "properties": {
+                                        "question": {
+                                            "bsonType": "string",
+                                            "description": "Poll question",
+                                        },
+                                        "options": {
+                                            "bsonType": "array",
+                                            "description": "List of poll options",
+                                            "items": {"bsonType": "string"},
+                                        },
+                                        "responses": {
+                                            "bsonType": "object",
+                                            "description": "Responses to the poll",
+                                            "additionalProperties": {"bsonType": "int"},
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    "post_class": {
+                        "bsonType": "object",
+                        "description": "Post-class segment data",
+                        "properties": {
+                            "assignments": {
+                                "bsonType": "array",
+                                "description": "List of assignments",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["id", "title", "due_date", "status"],
+                                    "properties": {
+                                        "id": {
+                                            "bsonType": "int",
+                                            "description": "Assignment ID",
+                                        },
+                                        "title": {
+                                            "bsonType": "string",
+                                            "description": "Title of the assignment",
+                                        },
+                                        "due_date": {
+                                            "bsonType": "date",
+                                            "description": "Due date of the assignment",
+                                        },
+                                        "status": {
+                                            "bsonType": "string",
+                                            "description": "Status of the assignment (e.g., pending, completed)",
+                                        },
+                                        "submissions": {
+                                            "bsonType": "array",
+                                            "description": "List of submissions",
+                                            "items": {
+                                                "bsonType": "object",
+                                                "required": [
+                                                    "student_id",
+                                                    "file_url",
+                                                    "submitted_at",
+                                                ],
+                                                "properties": {
+                                                    "student_id": {
+                                                        "bsonType": "string",
+                                                        "description": "ID of the student who submitted the assignment",
+                                                    },
+                                                    "file_url": {
+                                                        "bsonType": "string",
+                                                        "description": "URL of the submitted file",
+                                                    },
+                                                    "submitted_at": {
+                                                        "bsonType": "date",
+                                                        "description": "Date when the assignment was submitted",
+                                                    },
+                                                },
+                                            },
+                                        },
+                                    },
+                                },
+                            }
+                        },
+                    },
+                },
+            },
+        },
+    },
+}
+
+# Create the collection with the schema
+# db.create_collection("courses_collection2", validator={"$jsonSchema": course_schema})
+
+# sample_course = {
+#     "course_id": "CS101",
+#     "title": "Introduction to Computer Science",
+#     "description": "This course covers the basics of computer science and programming.",
+#     "faculty": "Dr. John Doe",
+#     "faculty_id": "F101",
+#     "duration": "10 weeks",
+#     "created_at": datetime.utcnow(),
+#     "sessions": [
+#         {
+#             "session_id": "S101",
+#             "title": "Introduction to Programming Fundamentals",
+#             "date": datetime.utcnow() - timedelta(days=7),
+#             "status": "completed",
+#             "created_at": datetime.utcnow() - timedelta(days=7),
+#             "pre_class": {
+#                 "resources": [
+#                     {
+#                         "type": "pdf",
+#                         "title": "Introduction to Python Basics",
+#                         "url": "/assets/python_basics.pdf",
+#                         "vector": [0.1, 0.2, 0.3]  # Example vector
+#                     }
+#                 ],
+#                 "completion_required": True
+#             },
+#             "in_class": {
+#                 "topics": ["Variables", "Data Types", "Basic Operations"],
+#                 "quiz": {
+#                     "title": "Python Basics Quiz",
+#                     "questions": 5,
+#                     "duration": 15
+#                 },
+#                 "polls": [
+#                     {
+#                         "question": "How comfortable are you with Python syntax?",
+#                         "options": ["Very", "Somewhat", "Not at all"],
+#                         "responses": {"Very": 10, "Somewhat": 5, "Not at all": 2}
+#                     }
+#                 ]
+#             },
+#             "post_class": {
+#                 "assignments": [
+#                     {
+#                         "id": 1,
+#                         "title": "Basic Python Programs",
+#                         "due_date": datetime.utcnow() + timedelta(days=2),
+#                         "status": "pending",
+#                         "submissions": []
+#                     }
+#                 ]
+#             }
+#         },
+#         {
+#             "session_id": "S102",
+#             "title": "Control Flow and Functions",
+#             "date": datetime.utcnow() - timedelta(days=3),
+#             "status": "completed",
+#             "created_at": datetime.utcnow() - timedelta(days=3),
+#             "pre_class": {
+#                 "resources": [
+#                     {
+#                         "type": "pdf",
+#                         "title": "Control Flow in Python",
+#                         "url": "/assets/control_flow.pdf",
+#                         "vector": [0.4, 0.5, 0.6]  # Example vector
+#                     }
+#                 ],
+#                 "completion_required": True
+#             },
+#             "in_class": {
+#                 "topics": ["If-else statements", "Loops", "Function definitions"],
+#                 "quiz": {
+#                     "title": "Control Flow Quiz",
+#                     "questions": 8,
+#                     "duration": 20
+#                 },
+#                 "polls": [
+#                     {
+#                         "question": "Which loop type do you find more intuitive?",
+#                         "options": ["For loops", "While loops", "Both"],
+#                         "responses": {"For loops": 12, "While loops": 8, "Both": 10}
+#                     }
+#                 ]
+#             },
+#             "post_class": {
+#                 "assignments": [
+#                     {
+#                         "id": 2,
+#                         "title": "Function Implementation Exercise",
+#                         "due_date": datetime.utcnow() + timedelta(days=4),
+#                         "status": "pending",
+#                         "submissions": []
+#                     }
+#                 ]
+#             }
+#         }
+#     ]
+# }
+courses_collection2 = db["courses_collection2"]
+
+
+# Define the users schema
+users_schema = {
+    "bsonType": "object",
+    "required": ["user_id", "username", "password", "role", "created_at"],
+    "properties": {
+        "user_id": {
+            "bsonType": "string",
+            "description": "Unique identifier for the user",
+        },
+        "username": {"bsonType": "string", "description": "Name of the User"},
+        "password": {"bsonType": "string", "description": "Password of the user"},
+        "role": {
+            "bsonType": "string",
+            "description": "Type of user (e.g., student, faculty)",
+        },
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the user was created",
+        },
+    },
+}
+# Create the collection with the schema
+# db.create_collection("users", validator={"$jsonSchema": users_schema})
+users_collection = db["users"]
+
+
+# Defining the Student Collection
+student_schema = {
+    "bsonType": "object",
+    "required": ["SID", "full_name", "password", "enrolled_courses", "created_at"],
+    "properties": {
+        "SID": {
+            "bsonType": "string",
+            "description": "Unique identifier for the student",
+        },
+        "full_name": {"bsonType": "string", "description": "Full name of the student"},
+        "password": {
+            "bsonType": "string",
+            "description": "Hashed password of the student",
+        },
+        "enrolled_courses": {
+            "bsonType": "array",
+            "description": "List of courses the student is enrolled in",
+            "items": {
+                "bsonType": "object",
+                "required": ["course_id", "title"],
+                "properties": {
+                    "course_id": {
+                        "bsonType": "string",
+                        "description": "Unique identifier for the course",
+                    },
+                    "title": {
+                        "bsonType": "string",
+                        "description": "Title of the course",
+                    },
+                },
+            },
+        },
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the student was created",
+        },
+    },
+}
+# Defining the Faculty Collection
+faculty_schema = {
+    "bsonType": "object",
+    "required": ["TID", "full_name", "password", "courses_taught", "created_at"],
+    "properties": {
+        "TID": {
+            "bsonType": "string",
+            "description": "Unique identifier for the faculty",
+        },
+        "full_name": {"bsonType": "string", "description": "Full name of the faculty"},
+        "password": {
+            "bsonType": "string",
+            "description": "Hashed password of the faculty",
+        },
+        "courses_taught": {
+            "bsonType": "array",
+            "description": "List of courses the faculty is teaching",
+            "items": {
+                "bsonType": "object",
+                "required": ["course_id", "title"],
+                "properties": {
+                    "course_id": {
+                        "bsonType": "string",
+                        "description": "Unique identifier for the course",
+                    },
+                    "title": {
+                        "bsonType": "string",
+                        "description": "Title of the course",
+                    },
+                },
+            },
+        },
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the faculty was created",
+        },
+    },
+}
+# Creating the Collections
+# db.create_collection("students", validator={"$jsonSchema": student_schema})
+# db.create_collection("faculty", validator={"$jsonSchema": faculty_schema})
+
+students_collection = db["students"]
+faculty_collection = db["faculty"]
+
+# Defining the Vector Collection Schema
+vector_schema = {
+    "bsonType": "object",
+    "required": ["resource_id", "vector"],
+    "properties": {
+        "resource_id": {
+            "bsonType": "objectId",
+            "description": "Unique identifier for the resource",
+        },
+        "vector": {
+            "bsonType": "array",
+            "description": "Vector representation of the resource",
+            "items": {"bsonType": "double"},
+        },
+        "text": {"bsonType": "string", "description": "Text content of the resource"},
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the vector was created",
+        },
+    },
+}
+# Creating the Vector Collection
+# db.create_collection("vectors", validator={"$jsonSchema": vector_schema})
+vectors_collection = db["vectors"]
+
+
+# Creating a Chat-History Collection
+# Creating a Chat-History Collection
+chat_history_schema = {
+    "bsonType": "object",
+    "required": ["user_id", "session_id", "messages", "timestamp"],
+    "properties": {
+        "user_id": {
+            "bsonType": "objectId",
+            "description": "Unique identifier for the user",
+        },
+        "session_id": {
+            "bsonType": "string",
+            "description": "Identifier for the session",
+        },
+        "timestamp": {
+            "bsonType": "date",
+            "description": "Timestamp when the chat session started",
+        },
+        "messages": {
+            "bsonType": "array",
+            "description": "List of chat messages",
+            "items": {
+                "bsonType": "object",
+                "properties": {
+                    "prompt": {
+                        "bsonType": "string",
+                        "description": "User's question or prompt",
+                    },
+                    "response": {
+                        "bsonType": "string",
+                        "description": "Assistant's response",
+                    },
+                    "timestamp": {
+                        "bsonType": "date",
+                        "description": "Timestamp of the message",
+                    },
+                },
+            },
+        },
+    },
+}
+
+# Create the collection with the schema
+# db.create_collection("chat_history", validator={"$jsonSchema": chat_history_schema})
+chat_history_collection = db["chat_history"]
+
+
+# Database setup for Research Assistant
+# Research Assistant Schema
+research_assistant_schema = {
+    "bsonType": "object",
+    "required": ["full_name", "password", "email", "courses_assisted"],
+    "properties": {
+        "full_name": {
+            "bsonType": "string",
+            "description": "Full name of the research assistant",
+        },
+        "password": {
+            "bsonType": "string",
+            "description": "Hashed password of the research assistant",
+        },
+        "email": {
+            "bsonType": "string",
+            "description": "Email address of the research assistant",
+        },
+        "courses_assisted": {
+            "bsonType": "array",
+            "description": "List of courses the research assistant is assisting",
+            "items": {
+                "bsonType": "object",
+                "required": ["course_id"],
+                "properties": {
+                    "course_id": {
+                        "bsonType": "string",
+                        "description": "ID of the course",
+                    }
+                },
+            },
+        },
+    },
+}
+
+# Create research assistants collection
+research_assistants_collection = db["research_assistants"]
+
+# Create indexes
+research_assistants_collection.create_index("full_name", unique=True)
+research_assistants_collection.create_index("email", unique=True)
+
+
+# Optional: Sample data insertion function
+# def insert_sample_research_assistants():
+#     sample_research_assistants = [
+#         {
+#             "full_name": "John Doe RA",
+#             "password": generate_password_hash("password123"),
+#             "email": "[email protected]",
+#             "courses_assisted": [{"course_id": "CS101"}, {"course_id": "CS102"}],
+#         }
+#     ]
+
+#     try:
+#         research_assistants_collection.insert_many(sample_research_assistants)
+#         print("Sample research assistants inserted successfully!")
+#     except Exception as e:
+#         print(f"Error inserting sample research assistants: {e}")
+
+# if __name__ == "__main__":
+#     insert_sample_analysts()

entire_download.py

@@ -0,0 +1,90 @@
+import streamlit as st
+import pandas as pd
+from pymongo import MongoClient
+from dotenv import load_dotenv
+import os
+
+# 1. Load environment variables
+load_dotenv()
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+
+# 2. Create MongoDB connection
+client = MongoClient(MONGODB_URI)
+db = client["novascholar_db"]
+collection = db["research_papers"]
+
+
+def get_collection_data(paper_type: str):
+    """
+    Fetch all documents from the specified collection based on paper type.
+    """
+    try:
+        # Determine collection name based on paper type
+        collection_name = paper_type.replace(" ", "_").lower()
+        doc_collection = db[collection_name]
+
+        # Get all documents
+        docs = list(doc_collection.find())
+
+        # Convert ObjectId to string
+        for doc in docs:
+            doc["_id"] = str(doc["_id"])
+
+        return docs
+    except Exception as e:
+        st.error(f"Database Error: {str(e)}")
+        return None
+
+
+def main():
+    st.title("MongoDB Collection Download")
+    st.write("Download all documents from the selected research paper collection")
+
+    # Dropdown to select the type of research paper
+    paper_type = st.selectbox(
+        "Select type of research paper:",
+        [
+            "Review Based Paper",
+            "Opinion/Perspective Based Paper",
+            "Empirical Research Paper",
+            "Research Paper (Other)",
+        ],
+    )
+
+    if st.button("Fetch Data"):
+        with st.spinner("Retrieving documents from MongoDB..."):
+            docs = get_collection_data(paper_type)
+
+            if docs:
+                # Convert to DataFrame
+                df = pd.DataFrame(docs)
+                # Convert lists to comma-separated strings for consistency
+                for col in df.columns:
+                    if df[col].apply(lambda x: isinstance(x, list)).any():
+                        df[col] = df[col].apply(
+                            lambda x: (
+                                ", ".join(map(str, x)) if isinstance(x, list) else x
+                            )
+                        )
+                st.success(
+                    f"Successfully retrieved {len(df)} documents from '{paper_type}' collection."
+                )
+                st.dataframe(df)
+
+                # Provide option to download the data as CSV
+                csv = df.to_csv(index=False).encode("utf-8")
+                st.download_button(
+                    label="Download CSV",
+                    data=csv,
+                    file_name=f"{paper_type.replace(' ', '_').lower()}_papers.csv",
+                    mime="text/csv",
+                )
+            else:
+                st.warning(f"No documents found in the '{paper_type}' collection.")
+
+
+if __name__ == "__main__":
+    main()

extract.py

@@ -0,0 +1,140 @@
+import streamlit as st
+import pandas as pd
+import PyPDF2
+import io
+import os
+from dotenv import load_dotenv
+import requests
+import time
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+
+def call_perplexity_api(prompt: str) -> str:
+    """Call Perplexity AI with a prompt, return the text response if successful."""
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.3,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        st.error(f"API Error: {str(e)}")
+        return ""
+
+def extract_text_from_pdf(pdf_file):
+    """Extract text content from a PDF file."""
+    pdf_reader = PyPDF2.PdfReader(pdf_file)
+    text = ""
+    for page in pdf_reader.pages:
+        text += page.extract_text() + "\n"
+    return text
+
+def analyze_paper(text: str, category: str) -> str:
+    """Generate a prompt and get analysis for a specific category."""
+    prompts = {
+        "Summarized Abstract": "Extract and summarize the abstract from this research paper:",
+        "Results": "What are the main results and findings from this research paper:",
+        "Summarized Introduction": "Summarize the introduction section of this research paper:",
+        "Methods Used": "What are the main methods and methodologies used in this research:",
+        "Literature Survey": "Summarize the literature review or related work from this paper:",
+        "Limitations": "What are the limitations mentioned in this research:",
+        "Contributions": "What are the main contributions of this research:",
+        "Practical Implications": "What are the practical implications of this research:",
+        "Objectives": "What are the main objectives of this research:",
+        "Findings": "What are the key findings from this research:",
+        "Future Research": "What future research directions are suggested in this paper:",
+        "Dependent Variables": "What are the dependent variables studied in this research:",
+        "Independent Variables": "What are the independent variables studied in this research:",
+        "Dataset": "What dataset(s) were used in this research:",
+        "Problem Statement": "What is the main problem statement or research question:",
+        "Challenges": "What challenges were faced or addressed in this research:",
+        "Applications": "What are the potential applications of this research:"
+    }
+    
+    prompt = f"{prompts[category]}\n\nPaper text: {text[:5000]}"  # Limit text to avoid token limits
+    return call_perplexity_api(prompt)
+
+def main():
+    st.title("Research Paper Analysis Tool")
+    
+    # File uploader
+    uploaded_files = st.file_uploader("Upload PDF files", type="pdf", accept_multiple_files=True)
+    
+    if uploaded_files:
+        if st.button("Process Papers"):
+            # Initialize progress bar
+            progress_bar = st.progress(0)
+            status_text = st.empty()
+            
+            # Initialize results dictionary
+            results = []
+            
+            # Define categories
+            categories = [
+                "Summarized Abstract", "Results", "Summarized Introduction", 
+                "Methods Used", "Literature Survey", "Limitations", 
+                "Contributions", "Practical Implications", "Objectives",
+                "Findings", "Future Research", "Dependent Variables",
+                "Independent Variables", "Dataset", "Problem Statement",
+                "Challenges", "Applications"
+            ]
+            
+            # Process each file
+            for i, file in enumerate(uploaded_files):
+                status_text.text(f"Processing {file.name}...")
+                
+                # Extract text from PDF
+                text = extract_text_from_pdf(file)
+                
+                # Initialize paper results
+                paper_results = {"Filename": file.name}
+                
+                # Analyze each category
+                for j, category in enumerate(categories):
+                    status_text.text(f"Processing {file.name} - {category}")
+                    paper_results[category] = analyze_paper(text, category)
+                    
+                    # Update progress
+                    progress = (i * len(categories) + j + 1) / (len(uploaded_files) * len(categories))
+                    progress_bar.progress(progress)
+                    
+                    # Add small delay to avoid API rate limits
+                    time.sleep(1)
+                
+                results.append(paper_results)
+            
+            # Create DataFrame
+            df = pd.DataFrame(results)
+            
+            # Convert DataFrame to CSV
+            csv = df.to_csv(index=False)
+            
+            # Create download button
+            st.download_button(
+                label="Download Results as CSV",
+                data=csv,
+                file_name="research_papers_analysis.csv",
+                mime="text/csv"
+            )
+            
+            # Display results in the app
+            st.subheader("Analysis Results")
+            st.dataframe(df)
+            
+            status_text.text("Processing complete!")
+            progress_bar.progress(1.0)
+
+if __name__ == "__main__":
+    main()

file_upload_vectorize.py

@@ -1,179 +1,179 @@
-from pymongo import MongoClient
-from datetime import datetime
-import openai
-import google.generativeai as genai
-import streamlit as st
-from db import courses_collection2, faculty_collection, students_collection, vectors_collection
-from PIL import Image
-import PyPDF2, docx, io
-from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Document
-from bson import ObjectId
-from dotenv import load_dotenv
-import os
-from create_course import courses_collection
-
-load_dotenv()
-MONGO_URI = os.getenv('MONGO_URI')
-OPENAI_KEY = os.getenv('OPENAI_KEY')
-GEMINI_KEY = os.getenv('GEMINI_KEY')
-
-
-client = MongoClient(MONGO_URI)
-db = client['novascholar_db']
-resources_collection = db['resources']
-
-# Configure APIs
-openai.api_key = OPENAI_KEY
-genai.configure(api_key=GEMINI_KEY)
-model = genai.GenerativeModel('gemini-pro')
-
-def upload_resource(course_id, session_id, file_name, file_content, material_type):
-    # material_data = {
-    #     "session_id": session_id,
-    #     "course_id": course_id,
-    #     "file_name": file_name,
-    #     "file_content": file_content,
-    #     "material_type": material_type,
-    #     "uploaded_at": datetime.utcnow()
-    # }
-    # return resources_collection.insert_one(material_data)
-    # resource_id = ObjectId()
-    
-    # Extract text content from the file
-    text_content = extract_text_from_file(file_content)
-    
-    # Check if a resource with this file name already exists
-    existing_resource = resources_collection.find_one({
-        "session_id": session_id,
-        "file_name": file_name
-    })
-    
-    if existing_resource:
-        return existing_resource["_id"]
-
-    # Read the file content
-    file_content.seek(0)  # Reset the file pointer to the beginning
-    original_file_content = file_content.read()
-    
-
-    resource_data = {
-        "_id": ObjectId(),
-        "course_id": course_id,
-        "session_id": session_id,
-        "file_name": file_name,
-        "file_type": file_content.type,
-        "text_content": text_content,
-        "file_content": original_file_content,  # Store the original file content
-        "material_type": material_type,
-        "uploaded_at": datetime.utcnow()
-    }
-    
-    resources_collection.insert_one(resource_data)
-    resource_id = resource_data["_id"]
-    
-    courses_collection.update_one(
-        {
-            "course_id": course_id,
-            "sessions.session_id": session_id
-        },
-        {
-            "$push": {"sessions.$.pre_class.resources": resource_id}
-        }
-    )
-    # print("End of Upload Resource, Resource ID is: ", resource_id)
-    # return resource_id
-    if text_content: 
-        create_vector_store(text_content, resource_id)
-    return resource_id
-
-def assignment_submit(student_id, course_id, session_id, assignment_id,  file_name, file_content, text_content, material_type):
-    # Read the file content
-    file_content.seek(0)  # Reset the file pointer to the beginning
-    original_file_content = file_content.read()
-    
-    assignment_data = {
-        "student_id": student_id,
-        "course_id": course_id,
-        "session_id": session_id,
-        "assignment_id": assignment_id,
-        "file_name": file_name,
-        "file_type": file_content.type,
-        "file_content": original_file_content,  # Store the original file content
-        "text_content": text_content,
-        "material_type": material_type,
-        "submitted_at": datetime.utcnow(),
-        "file_url": "sample_url"
-    }
-    try:
-        courses_collection2.update_one(
-            {
-                "course_id": course_id,
-                "sessions.session_id": session_id,
-                "sessions.post_class.assignments.id": assignment_id
-            },
-            {
-                "$push": {"sessions.$.post_class.assignments.$[assignment].submissions": assignment_data}
-            },
-            array_filters=[{"assignment.id": assignment_id}]
-        )
-        return True
-    except Exception as db_error:
-        print(f"Error saving submission: {str(db_error)}")
-        return False
-
-def extract_text_from_file(uploaded_file):
-    text = ""
-    file_type = uploaded_file.type
-    
-    try:
-        if file_type == "text/plain":
-            text = uploaded_file.getvalue().decode("utf-8")
-        elif file_type == "application/pdf":
-            pdf_reader = PyPDF2.PdfReader(io.BytesIO(uploaded_file.getvalue()))
-            for page in pdf_reader.pages:
-                text += page.extract_text() + "\n"
-        elif file_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
-            doc = docx.Document(io.BytesIO(uploaded_file.getvalue()))
-            for para in doc.paragraphs:
-                text += para.text + "\n"
-        return text
-    except Exception as e:
-        st.error(f"Error processing file: {str(e)}")
-        return None
-
-def get_embedding(text):
-    response = openai.embeddings.create(
-        model="text-embedding-ada-002",
-        input=text
-    )
-    return response.data[0].embedding
-
-def create_vector_store(text, resource_id):
-    # resource_object_id = ObjectId(resource_id)
-    # Ensure resource_id is an ObjectId
-    # if not isinstance(resource_id, ObjectId):
-    #     resource_id = ObjectId(resource_id)
-    
-    existing_vector = vectors_collection.find_one({
-        "resource_id": resource_id,
-        "text": text
-    })
-    
-    if existing_vector:
-        print(f"Vector already exists for Resource ID: {resource_id}")
-        return
-
-    print(f"In Vector Store method, Resource ID is: {resource_id}")
-    document = Document(text=text)
-    embedding = get_embedding(text)
-    
-    vector_data = {
-        "resource_id": resource_id,
-        "vector": embedding,
-        "text": text,
-        "created_at": datetime.utcnow()
-    }
-    
-    vectors_collection.insert_one(vector_data)
-    
+from pymongo import MongoClient
+from datetime import datetime
+import openai
+import google.generativeai as genai
+import streamlit as st
+from db import courses_collection2, faculty_collection, students_collection, vectors_collection
+from PIL import Image
+import PyPDF2, docx, io
+from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Document
+from bson import ObjectId
+from dotenv import load_dotenv
+import os
+from create_course import courses_collection
+
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+OPENAI_KEY = os.getenv('OPENAI_KEY')
+GEMINI_KEY = os.getenv('GEMINI_KEY')
+
+
+client = MongoClient(MONGO_URI)
+db = client['novascholar_db']
+resources_collection = db['resources']
+
+# Configure APIs
+openai.api_key = OPENAI_KEY
+genai.configure(api_key=GEMINI_KEY)
+model = genai.GenerativeModel('gemini-pro')
+
+def upload_resource(course_id, session_id, file_name, file_content, material_type):
+    # material_data = {
+    #     "session_id": session_id,
+    #     "course_id": course_id,
+    #     "file_name": file_name,
+    #     "file_content": file_content,
+    #     "material_type": material_type,
+    #     "uploaded_at": datetime.utcnow()
+    # }
+    # return resources_collection.insert_one(material_data)
+    # resource_id = ObjectId()
+    
+    # Extract text content from the file
+    text_content = extract_text_from_file(file_content)
+    
+    # Check if a resource with this file name already exists
+    existing_resource = resources_collection.find_one({
+        "session_id": session_id,
+        "file_name": file_name
+    })
+    
+    if existing_resource:
+        return existing_resource["_id"]
+
+    # Read the file content
+    file_content.seek(0)  # Reset the file pointer to the beginning
+    original_file_content = file_content.read()
+    
+
+    resource_data = {
+        "_id": ObjectId(),
+        "course_id": course_id,
+        "session_id": session_id,
+        "file_name": file_name,
+        "file_type": file_content.type,
+        "text_content": text_content,
+        "file_content": original_file_content,  # Store the original file content
+        "material_type": material_type,
+        "uploaded_at": datetime.utcnow()
+    }
+    
+    resources_collection.insert_one(resource_data)
+    resource_id = resource_data["_id"]
+    
+    courses_collection.update_one(
+        {
+            "course_id": course_id,
+            "sessions.session_id": session_id
+        },
+        {
+            "$push": {"sessions.$.pre_class.resources": resource_id}
+        }
+    )
+    # print("End of Upload Resource, Resource ID is: ", resource_id)
+    # return resource_id
+    if text_content: 
+        create_vector_store(text_content, resource_id)
+    return resource_id
+
+def assignment_submit(student_id, course_id, session_id, assignment_id,  file_name, file_content, text_content, material_type):
+    # Read the file content
+    file_content.seek(0)  # Reset the file pointer to the beginning
+    original_file_content = file_content.read()
+    
+    assignment_data = {
+        "student_id": student_id,
+        "course_id": course_id,
+        "session_id": session_id,
+        "assignment_id": assignment_id,
+        "file_name": file_name,
+        "file_type": file_content.type,
+        "file_content": original_file_content,  # Store the original file content
+        "text_content": text_content,
+        "material_type": material_type,
+        "submitted_at": datetime.utcnow(),
+        "file_url": "sample_url"
+    }
+    try:
+        courses_collection2.update_one(
+            {
+                "course_id": course_id,
+                "sessions.session_id": session_id,
+                "sessions.post_class.assignments.id": assignment_id
+            },
+            {
+                "$push": {"sessions.$.post_class.assignments.$[assignment].submissions": assignment_data}
+            },
+            array_filters=[{"assignment.id": assignment_id}]
+        )
+        return True
+    except Exception as db_error:
+        print(f"Error saving submission: {str(db_error)}")
+        return False
+
+def extract_text_from_file(uploaded_file):
+    text = ""
+    file_type = uploaded_file.type
+    
+    try:
+        if file_type == "text/plain":
+            text = uploaded_file.getvalue().decode("utf-8")
+        elif file_type == "application/pdf":
+            pdf_reader = PyPDF2.PdfReader(io.BytesIO(uploaded_file.getvalue()))
+            for page in pdf_reader.pages:
+                text += page.extract_text() + "\n"
+        elif file_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+            doc = docx.Document(io.BytesIO(uploaded_file.getvalue()))
+            for para in doc.paragraphs:
+                text += para.text + "\n"
+        return text
+    except Exception as e:
+        st.error(f"Error processing file: {str(e)}")
+        return None
+
+def get_embedding(text):
+    response = openai.embeddings.create(
+        model="text-embedding-ada-002",
+        input=text
+    )
+    return response.data[0].embedding
+
+def create_vector_store(text, resource_id):
+    # resource_object_id = ObjectId(resource_id)
+    # Ensure resource_id is an ObjectId
+    # if not isinstance(resource_id, ObjectId):
+    #     resource_id = ObjectId(resource_id)
+    
+    existing_vector = vectors_collection.find_one({
+        "resource_id": resource_id,
+        "text": text
+    })
+    
+    if existing_vector:
+        print(f"Vector already exists for Resource ID: {resource_id}")
+        return
+
+    print(f"In Vector Store method, Resource ID is: {resource_id}")
+    document = Document(text=text)
+    embedding = get_embedding(text)
+    
+    vector_data = {
+        "resource_id": resource_id,
+        "vector": embedding,
+        "text": text,
+        "created_at": datetime.utcnow()
+    }
+    
+    vectors_collection.insert_one(vector_data)
+    
     # return VectorStoreIndex.from_documents([document])

gen_mcqs.py

@@ -1,206 +1,206 @@
-import ast
-from pymongo import MongoClient
-from datetime import datetime
-import openai
-import google.generativeai as genai
-from google.generativeai import GenerativeModel
-from dotenv import load_dotenv
-import os
-from file_upload_vectorize import resources_collection, vectors_collection, courses_collection2, faculty_collection
-
-# Load environment variables
-load_dotenv()
-MONGO_URI = os.getenv('MONGO_URI')
-OPENAI_KEY = os.getenv('OPENAI_KEY')
-GEMINI_KEY = os.getenv('GEMINI_KEY')
-
-# Configure APIs
-openai.api_key = OPENAI_KEY
-genai.configure(api_key=GEMINI_KEY)
-model = genai.GenerativeModel('gemini-pro')
-
-# Connect to MongoDB
-client = MongoClient(MONGO_URI)
-db = client['novascholar_db']
-quizzes_collection = db["quizzes"]
-
-def strip_code_markers(response_text):
-    """Strip off the markers ``` and python from a LLM model's response"""
-    if response_text.startswith("```python"):
-        response_text = response_text[len("```python"):].strip()
-    if response_text.startswith("```"):
-        response_text = response_text[len("```"):].strip()
-    if response_text.endswith("```"):
-        response_text = response_text[:-len("```")].strip()
-    return response_text
-
-
-# New function to generate MCQs using Gemini
-def generate_mcqs(context, num_questions, session_title, session_description):
-    """Generate MCQs either from context or session details"""
-    try:
-        # Initialize Gemini model
-        if context:
-            prompt = f"""
-            Based on the following content, generate {num_questions} multiple choice questions.
-            Format each question as a Python dictionary with the following structure:
-            {{
-                "question": "Question text here",
-                "options": ["A) option1", "B) option2", "C) option3", "D) option4"],
-                "correct_option": "A) option1" or "B) option2" or "C) option3" or "D) option4"
-            }}
-            
-            Content:
-            {context}
-            
-            Generate challenging but clear questions that test understanding of key concepts.
-            Return only the Python list of dictionaries.
-            """
-        else:
-            prompt = f"""
-            Generate {num_questions} multiple choice questions about the topic:
-            Title: {session_title}
-            Description: {session_description}
-            
-            Format each question as a Python dictionary with the following structure:
-            {{
-                "question": "Question text here",
-                "options": ["A) option1", "B) option2", "C) option3", "D) option4"],
-                "correct_option": "A" or "B" or "C" or "D"
-            }}
-            
-            Generate challenging but clear questions.
-            Return only the Python list of dictionaries without any additional formatting or markers
-            Do not write any other text, do not start the response with (```python), do not end the response with backticks(```)
-            A Sample response should look like this: Response Text: [
-                {
-                    "question": "Which of the following is NOT a valid data type in C++?",
-                    "options": ["int", "double", "boolean", "char"],
-                    "correct_option": "C"
-                }
-            ] (Notice that there are no backticks(```) around the response and no (```python)) 
-            .
-            """
-        
-        response = model.generate_content(prompt)
-        response_text = response.text.strip()
-        print("Response Text:", response_text)
-        modified_response_text = strip_code_markers(response_text)
-        print("Response Text Modified to:", modified_response_text)
-        # Extract and parse the response to get the list of MCQs
-        mcqs = ast.literal_eval(modified_response_text)  # Be careful with eval, consider using ast.literal_eval for production
-        print(mcqs)
-        if not mcqs:
-            raise ValueError("No questions generated")
-        return mcqs
-    except Exception as e:
-        print(f"Error generating MCQs: , error: {e}")
-        return None
-
-# New function to save quiz to database
-def save_quiz(course_id, session_id, title, questions, user_id):
-    """Save quiz to database"""
-    try:
-        quiz_data = {
-            "user_id": user_id,
-            "course_id": course_id,
-            "session_id": session_id,
-            "title": title,
-            "questions": questions,
-            "created_at": datetime.utcnow(),
-            "status": "active",
-            "submissions": []
-        }
-        result = quizzes_collection.insert_one(quiz_data)
-        return result.inserted_id
-    except Exception as e:
-        print(f"Error saving quiz: {e}")
-        return None
-    
-
-def get_student_quiz_score(quiz_id, student_id):
-    """Get student's score for a specific quiz"""
-    quiz = quizzes_collection.find_one(
-        {
-            "_id": quiz_id,
-            "submissions.student_id": student_id
-        },
-        {"submissions.$": 1}
-    )
-    if quiz and quiz.get('submissions'):
-        return quiz['submissions'][0].get('score')
-    return None
-
-# def submit_quiz_answers(quiz_id, student_id, student_answers):
-#     """Submit and score student's quiz answers"""
-#     quiz = quizzes_collection.find_one({"_id": quiz_id})
-#     if not quiz:
-#         return None
-    
-#     # Calculate score
-#     correct_answers = 0
-#     total_questions = len(quiz['questions'])
-    
-#     for q_idx, question in enumerate(quiz['questions']):
-#         if student_answers.get(str(q_idx)) == question['correct_option']:
-#             correct_answers += 1
-    
-#     score = (correct_answers / total_questions) * 100
-    
-#     # Store submission
-#     submission_data = {
-#         "student_id": student_id,
-#         "answers": student_answers,
-#         "score": score,
-#         "submitted_at": datetime.utcnow()
-#     }
-    
-#     # Update quiz with submission
-#     quizzes_collection.update_one(
-#         {"_id": quiz_id},
-#         {
-#             "$push": {"submissions": submission_data}
-#         }
-#     )
-    
-#     return score
-def submit_quiz_answers(quiz_id, student_id, student_answers):
-    """Submit and score student's quiz answers"""
-    try:
-        quiz = quizzes_collection.find_one({"_id": quiz_id})
-        if not quiz:
-            return None
-        
-        # Calculate score
-        correct_answers = 0
-        total_questions = len(quiz['questions'])
-        
-        for q_idx, question in enumerate(quiz['questions']):
-            student_answer = student_answers.get(str(q_idx))
-            if student_answer:  # Only check if answer was provided
-                # Extract the option letter (A, B, C, D) from the full answer string
-                answer_letter = student_answer.split(')')[0].strip()
-                if answer_letter == question['correct_option']:
-                    correct_answers += 1
-        
-        score = (correct_answers / total_questions) * 100
-        
-        # Store submission
-        submission_data = {
-            "student_id": student_id,
-            "answers": student_answers,
-            "score": score,
-            "submitted_at": datetime.utcnow()
-        }
-        
-        # Update quiz with submission
-        result = quizzes_collection.update_one(
-            {"_id": quiz_id},
-            {"$push": {"submissions": submission_data}}
-        )
-        
-        return score if result.modified_count > 0 else None
-        
-    except Exception as e:
-        print(f"Error submitting quiz: {e}")
+import ast
+from pymongo import MongoClient
+from datetime import datetime
+import openai
+import google.generativeai as genai
+from google.generativeai import GenerativeModel
+from dotenv import load_dotenv
+import os
+from file_upload_vectorize import resources_collection, vectors_collection, courses_collection2, faculty_collection
+
+# Load environment variables
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+OPENAI_KEY = os.getenv('OPENAI_KEY')
+GEMINI_KEY = os.getenv('GEMINI_KEY')
+
+# Configure APIs
+openai.api_key = OPENAI_KEY
+genai.configure(api_key=GEMINI_KEY)
+model = genai.GenerativeModel('gemini-pro')
+
+# Connect to MongoDB
+client = MongoClient(MONGO_URI)
+db = client['novascholar_db']
+quizzes_collection = db["quizzes"]
+
+def strip_code_markers(response_text):
+    """Strip off the markers ``` and python from a LLM model's response"""
+    if response_text.startswith("```python"):
+        response_text = response_text[len("```python"):].strip()
+    if response_text.startswith("```"):
+        response_text = response_text[len("```"):].strip()
+    if response_text.endswith("```"):
+        response_text = response_text[:-len("```")].strip()
+    return response_text
+
+
+# New function to generate MCQs using Gemini
+def generate_mcqs(context, num_questions, session_title, session_description):
+    """Generate MCQs either from context or session details"""
+    try:
+        # Initialize Gemini model
+        if context:
+            prompt = f"""
+            Based on the following content, generate {num_questions} multiple choice questions.
+            Format each question as a Python dictionary with the following structure:
+            {{
+                "question": "Question text here",
+                "options": ["A) option1", "B) option2", "C) option3", "D) option4"],
+                "correct_option": "A) option1" or "B) option2" or "C) option3" or "D) option4"
+            }}
+            
+            Content:
+            {context}
+            
+            Generate challenging but clear questions that test understanding of key concepts.
+            Return only the Python list of dictionaries.
+            """
+        else:
+            prompt = f"""
+            Generate {num_questions} multiple choice questions about the topic:
+            Title: {session_title}
+            Description: {session_description}
+            
+            Format each question as a Python dictionary with the following structure:
+            {{
+                "question": "Question text here",
+                "options": ["A) option1", "B) option2", "C) option3", "D) option4"],
+                "correct_option": "A" or "B" or "C" or "D"
+            }}
+            
+            Generate challenging but clear questions.
+            Return only the Python list of dictionaries without any additional formatting or markers
+            Do not write any other text, do not start the response with (```python), do not end the response with backticks(```)
+            A Sample response should look like this: Response Text: [
+                {
+                    "question": "Which of the following is NOT a valid data type in C++?",
+                    "options": ["int", "double", "boolean", "char"],
+                    "correct_option": "C"
+                }
+            ] (Notice that there are no backticks(```) around the response and no (```python)) 
+            .
+            """
+        
+        response = model.generate_content(prompt)
+        response_text = response.text.strip()
+        print("Response Text:", response_text)
+        modified_response_text = strip_code_markers(response_text)
+        print("Response Text Modified to:", modified_response_text)
+        # Extract and parse the response to get the list of MCQs
+        mcqs = ast.literal_eval(modified_response_text)  # Be careful with eval, consider using ast.literal_eval for production
+        print(mcqs)
+        if not mcqs:
+            raise ValueError("No questions generated")
+        return mcqs
+    except Exception as e:
+        print(f"Error generating MCQs: , error: {e}")
+        return None
+
+# New function to save quiz to database
+def save_quiz(course_id, session_id, title, questions, user_id):
+    """Save quiz to database"""
+    try:
+        quiz_data = {
+            "user_id": user_id,
+            "course_id": course_id,
+            "session_id": session_id,
+            "title": title,
+            "questions": questions,
+            "created_at": datetime.utcnow(),
+            "status": "active",
+            "submissions": []
+        }
+        result = quizzes_collection.insert_one(quiz_data)
+        return result.inserted_id
+    except Exception as e:
+        print(f"Error saving quiz: {e}")
+        return None
+    
+
+def get_student_quiz_score(quiz_id, student_id):
+    """Get student's score for a specific quiz"""
+    quiz = quizzes_collection.find_one(
+        {
+            "_id": quiz_id,
+            "submissions.student_id": student_id
+        },
+        {"submissions.$": 1}
+    )
+    if quiz and quiz.get('submissions'):
+        return quiz['submissions'][0].get('score')
+    return None
+
+# def submit_quiz_answers(quiz_id, student_id, student_answers):
+#     """Submit and score student's quiz answers"""
+#     quiz = quizzes_collection.find_one({"_id": quiz_id})
+#     if not quiz:
+#         return None
+    
+#     # Calculate score
+#     correct_answers = 0
+#     total_questions = len(quiz['questions'])
+    
+#     for q_idx, question in enumerate(quiz['questions']):
+#         if student_answers.get(str(q_idx)) == question['correct_option']:
+#             correct_answers += 1
+    
+#     score = (correct_answers / total_questions) * 100
+    
+#     # Store submission
+#     submission_data = {
+#         "student_id": student_id,
+#         "answers": student_answers,
+#         "score": score,
+#         "submitted_at": datetime.utcnow()
+#     }
+    
+#     # Update quiz with submission
+#     quizzes_collection.update_one(
+#         {"_id": quiz_id},
+#         {
+#             "$push": {"submissions": submission_data}
+#         }
+#     )
+    
+#     return score
+def submit_quiz_answers(quiz_id, student_id, student_answers):
+    """Submit and score student's quiz answers"""
+    try:
+        quiz = quizzes_collection.find_one({"_id": quiz_id})
+        if not quiz:
+            return None
+        
+        # Calculate score
+        correct_answers = 0
+        total_questions = len(quiz['questions'])
+        
+        for q_idx, question in enumerate(quiz['questions']):
+            student_answer = student_answers.get(str(q_idx))
+            if student_answer:  # Only check if answer was provided
+                # Extract the option letter (A, B, C, D) from the full answer string
+                answer_letter = student_answer.split(')')[0].strip()
+                if answer_letter == question['correct_option']:
+                    correct_answers += 1
+        
+        score = (correct_answers / total_questions) * 100
+        
+        # Store submission
+        submission_data = {
+            "student_id": student_id,
+            "answers": student_answers,
+            "score": score,
+            "submitted_at": datetime.utcnow()
+        }
+        
+        # Update quiz with submission
+        result = quizzes_collection.update_one(
+            {"_id": quiz_id},
+            {"$push": {"submissions": submission_data}}
+        )
+        
+        return score if result.modified_count > 0 else None
+        
+    except Exception as e:
+        print(f"Error submitting quiz: {e}")
         return None

goals2.py

@@ -1,658 +1,658 @@
-import streamlit as st
-from typing import List, Dict
-import httpx
-from pathlib import Path
-import os
-from dotenv import load_dotenv
-import json
-import numpy as np
-from pymongo import MongoClient
-from openai import OpenAI
-from datetime import datetime
-import asyncio
-import pandas as pd
-
-# Load environment variables
-load_dotenv()
-PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
-MONGODB_URI = os.getenv("MONGO_URI")
-OPENAI_API_KEY = os.getenv("OPENAI_KEY")
-
-# Initialize MongoDB client
-client = MongoClient(MONGODB_URI)
-db = client["document_analysis"]
-vectors_collection = db["document_vectors"]
-
-# Initialize OpenAI client
-openai_client = OpenAI(api_key=OPENAI_API_KEY)
-
-
-class GoalAnalyzer:
-    def __init__(self):
-        self.api_key = PERPLEXITY_API_KEY
-        self.base_url = "https://api.perplexity.ai/chat/completions"
-
-    def clean_json_string(self, content: str) -> str:
-        """Clean and extract valid JSON from string"""
-        # Remove markdown formatting
-        if "```json" in content:
-            content = content.split("```json")[1].split("```")[0]
-        elif "```" in content:
-            content = content.split("```")[1]
-
-        # Find the JSON object boundaries
-        start_idx = content.find("{")
-        end_idx = content.rfind("}") + 1
-
-        if start_idx != -1 and end_idx > 0:
-            content = content[start_idx:end_idx]
-
-        # Clean up common issues
-        content = content.strip()
-        content = content.replace("\n", "")
-        content = content.replace("'", '"')
-
-        return content
-
-    async def get_perplexity_analysis(self, text: str, goal: str) -> Dict:
-        """Get analysis from Perplexity API"""
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
-        }
-
-        prompt = f"""
-        Analyze the following text in context of the goal: {goal}
-        
-        Text: {text}
-        
-        Provide analysis in the following JSON format:
-        {{
-            "themes": ["theme1", "theme2"],
-            "subthemes": {{"theme1": ["subtheme1", "subtheme2"], "theme2": ["subtheme3"]}},
-            "keywords": ["keyword1", "keyword2"],
-            "relevance_score": 0-100
-        }}
-        """
-
-        try:
-            async with httpx.AsyncClient() as client:
-                payload = {
-                    "model": "llama-3.1-sonar-small-128k-chat",  # Updated to supported model
-                    "messages": [
-                        {
-                            "role": "system",
-                            "content": "You are an AI assistant that analyzes documents and provides structured analysis.",
-                        },
-                        {"role": "user", "content": prompt},
-                    ],
-                    "max_tokens": 1024,
-                }
-
-                # Debug info using expander
-                with st.expander("Debug Info", expanded=False):
-                    st.write("Request payload:", payload)
-
-                response = await client.post(
-                    self.base_url, headers=headers, json=payload, timeout=30.0
-                )
-
-                # Debug response info
-                with st.expander("Response Info", expanded=False):
-                    st.write("Response status:", response.status_code)
-                    st.write("Response headers:", dict(response.headers))
-                    st.write("Response content:", response.text)
-
-                if response.status_code != 200:
-                    error_detail = (
-                        response.json() if response.content else "No error details"
-                    )
-                    raise Exception(
-                        f"API returned status code {response.status_code}. Details: {error_detail}"
-                    )
-
-                result = response.json()
-                content = (
-                    result.get("choices", [{}])[0].get("message", {}).get("content", "")
-                )
-
-                # Clean and parse JSON
-                cleaned_content = self.clean_json_string(content)
-
-                try:
-                    analysis = json.loads(cleaned_content)
-
-                    # Validate required fields
-                    required_fields = [
-                        "themes",
-                        "subthemes",
-                        "keywords",
-                        "relevance_score",
-                    ]
-                    for field in required_fields:
-                        if field not in analysis:
-                            analysis[field] = [] if field != "relevance_score" else 0
-
-                    return analysis
-
-                except json.JSONDecodeError as e:
-                    st.error(f"JSON parsing error: {str(e)}")
-                    st.error(f"Failed content: {cleaned_content}")
-                    return {
-                        "themes": ["Error parsing themes"],
-                        "subthemes": {"Error": ["Failed to parse subthemes"]},
-                        "keywords": ["parsing-error"],
-                        "relevance_score": 0,
-                    }
-
-        except Exception as e:
-            st.error(f"API Error: {str(e)}")
-            return None
-
-    def extract_text_from_file(self, file) -> str:
-        """Extract text content from uploaded file"""
-        try:
-            text = ""
-            file_type = file.type
-
-            if file_type == "text/plain":
-                text = file.getvalue().decode("utf-8")
-            elif file_type == "application/pdf":
-                import PyPDF2
-
-                pdf_reader = PyPDF2.PdfReader(file)
-                for page in pdf_reader.pages:
-                    text += page.extract_text()
-            elif (
-                file_type
-                == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-            ):
-                import docx
-
-                doc = docx.Document(file)
-                text = " ".join([paragraph.text for paragraph in doc.paragraphs])
-
-            return text
-        except Exception as e:
-            st.error(f"Error extracting text: {str(e)}")
-            return ""
-
-
-class DocumentVectorizer:
-    def __init__(self):
-        self.model = "text-embedding-ada-002"
-        self.client = MongoClient(MONGODB_URI)
-        self.db = self.client["document_analysis"]
-        self.vectors_collection = self.db["document_vectors"]
-
-        # Create vector search index if it doesn't exist
-        try:
-            self.vectors_collection.create_index(
-                [("vector", "2dsphere")],  # Changed to 2dsphere for vector indexing
-                {
-                    "vectorSearchConfig": {
-                        "dimensions": 1536,  # OpenAI embedding dimensions
-                        "similarity": "cosine",
-                    }
-                },
-            )
-        except Exception as e:
-            st.warning(f"Vector index may already exist")
-
-    def get_embedding(self, text: str) -> list:
-        """Get embedding vector for text using OpenAI"""
-        try:
-            response = openai_client.embeddings.create(model=self.model, input=text)
-            return response.data[0].embedding
-        except Exception as e:
-            st.error(f"Error getting embedding: {str(e)}")
-            return None
-
-    # Add this method to DocumentVectorizer class
-    def vector_exists(self, doc_name: str) -> bool:
-        """Check if vector exists for document"""
-        return self.vectors_collection.count_documents({"name": doc_name}) > 0
-
-    # Update store_vector method in DocumentVectorizer class
-    def store_vector(self, doc_name: str, vector: list, text: str, goal: str = None):
-        """Store document/goal vector in MongoDB using upsert"""
-        try:
-            vector_doc = {
-                "name": doc_name,
-                "vector": vector,
-                "text": text,
-                "type": "document" if goal is None else "goal",
-                "goal": goal,
-                "updated_at": datetime.utcnow(),
-            }
-
-            # Use update_one with upsert
-            self.vectors_collection.update_one(
-                {"name": doc_name},
-                {"$set": vector_doc, "$setOnInsert": {"created_at": datetime.utcnow()}},
-                upsert=True,
-            )
-
-        except Exception as e:
-            st.error(f"Error storing vector: {str(e)}")
-
-    # Update vector_search method in DocumentVectorizer class
-    def vector_search(self, query_vector: List[float], limit: int = 5) -> List[Dict]:
-        """Search for similar documents using vector similarity"""
-        try:
-            # Get all documents
-            documents = list(self.vectors_collection.find({"type": "document"}))
-
-            # Calculate similarities
-            similarities = []
-            for doc in documents:
-                similarity = self.calculate_similarity(query_vector, doc["vector"])
-                similarities.append(
-                    {
-                        "name": doc["name"],
-                        "text": doc["text"],
-                        "similarity": similarity,  # Keep as float
-                        "similarity_display": f"{similarity*100:.1f}%",  # Add display version
-                    }
-                )
-
-            # Sort by similarity and get top k
-            sorted_docs = sorted(
-                similarities,
-                key=lambda x: x["similarity"],  # Sort by float value
-                reverse=True,
-            )[:limit]
-
-            return sorted_docs
-
-        except Exception as e:
-            st.error(f"Vector search error: {str(e)}")
-            return []
-
-    def find_similar_documents(self, text: str, limit: int = 5) -> List[Dict]:
-        """Find similar documents for given text"""
-        vector = self.get_embedding(text)
-        if vector:
-            return self.vector_search(vector, limit)
-        return []
-
-    def calculate_similarity(self, vector1: list, vector2: list) -> float:
-        """Calculate cosine similarity between two vectors"""
-        return np.dot(vector1, vector2) / (
-            np.linalg.norm(vector1) * np.linalg.norm(vector2)
-        )
-
-
-def display_analysis_results(analysis: Dict):
-    """Display analysis results in Streamlit UI"""
-    if not analysis:
-        return
-
-    # Display Themes
-    st.subheader("Themes")
-    for theme in analysis.get("themes", []):
-        with st.expander(f"🎯 {theme}"):
-            # Display subthemes for this theme
-            subthemes = analysis.get("subthemes", {}).get(theme, [])
-            if subthemes:
-                st.write("**Subthemes:**")
-                for subtheme in subthemes:
-                    st.write(f"- {subtheme}")
-
-    # Display Keywords
-    st.subheader("Keywords")
-    keywords = analysis.get("keywords", [])
-    st.write(" | ".join([f"🔑 {keyword}" for keyword in keywords]))
-
-    # Display Relevance Score
-    score = analysis.get("relevance_score", 0)
-    st.metric("Relevance Score", f"{score}%")
-
-
-def display_analyst_dashboard():
-    st.title("Multi-Goal Document Analysis")
-
-    with st.sidebar:
-        st.markdown("### Input Section")
-        tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
-        # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
-
-        with tab1:
-            # Multiple goals input
-            num_goals = st.number_input("Number of goals:", min_value=1, value=1)
-            goals = []
-            for i in range(num_goals):
-                goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100)
-                if goal:
-                    goals.append(goal)
-
-            uploaded_files = st.file_uploader(
-                "Upload documents",
-                accept_multiple_files=True,
-                type=["txt", "pdf", "docx"],
-            )
-            analyze_button = (
-                st.button("Analyze Documents") if goals and uploaded_files else None
-            )
-
-        with tab2:
-            # Keep existing similarity search tab
-            search_text = st.text_area("Enter text to find similar documents:")
-            search_limit = st.slider("Number of results", 1, 10, 5)
-            search_button = st.button("Search Similar") if search_text else None
-
-        if st.button("Logout", use_container_width=True):
-            for key in st.session_state.keys():
-                del st.session_state[key]
-            st.rerun()
-
-    if analyze_button:
-        analyzer = GoalAnalyzer()
-        vectorizer = DocumentVectorizer()
-
-        # Store vectors
-        doc_vectors = {}
-        goal_vectors = {}
-
-        # Process goals first
-        with st.spinner("Processing goals..."):
-            for i, goal in enumerate(goals):
-                vector = vectorizer.get_embedding(goal)
-                if vector:
-                    goal_vectors[f"Goal {i+1}"] = vector
-                    vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal)
-
-        # Process documents
-        with st.spinner("Processing documents..."):
-            for file in uploaded_files:
-                st.markdown(f"### Analysis for {file.name}")
-
-                if vectorizer.vector_exists(file.name):
-                    st.info(f"Vector already exists for {file.name}")
-                    existing_doc = vectorizer.vectors_collection.find_one(
-                        {"name": file.name}
-                    )
-                    doc_vectors[file.name] = existing_doc["vector"]
-                else:
-                    text = analyzer.extract_text_from_file(file)
-                    if not text:
-                        st.warning(f"Could not extract text from {file.name}")
-                        continue
-
-                    vector = vectorizer.get_embedding(text)
-                    if vector:
-                        doc_vectors[file.name] = vector
-                        vectorizer.store_vector(file.name, vector, text)
-
-                # Display goal similarities
-                st.subheader("Goal Relevance Scores")
-                col1, col2 = st.columns([1, 2])
-
-                with col1:
-                    for goal_name, goal_vector in goal_vectors.items():
-                        similarity = (
-                            vectorizer.calculate_similarity(
-                                doc_vectors[file.name], goal_vector
-                            )
-                            * 100
-                        )
-                        st.metric(f"{goal_name}", f"{similarity:.1f}%")
-
-                with col2:
-                    # Get analysis for all goals combined
-                    analysis = asyncio.run(
-                        analyzer.get_perplexity_analysis(text, " | ".join(goals))
-                    )
-                    display_analysis_results(analysis)
-
-                st.divider()
-
-            # Document similarity matrix
-            if len(doc_vectors) > 1:
-                st.markdown("### Document Similarity Matrix")
-                files = list(doc_vectors.keys())
-                similarity_matrix = []
-
-                for file1 in files:
-                    row = []
-                    for file2 in files:
-                        similarity = vectorizer.calculate_similarity(
-                            doc_vectors[file1], doc_vectors[file2]
-                        )
-                        row.append(similarity)
-                    similarity_matrix.append(row)
-
-                df = pd.DataFrame(similarity_matrix, columns=files, index=files)
-                st.dataframe(df.style.background_gradient(cmap="RdYlGn"))
-
-                # Add goal-document similarity matrix
-                st.markdown("### Goal-Document Similarity Matrix")
-                goal_doc_matrix = []
-                goal_names = list(goal_vectors.keys())
-
-                for file in files:
-                    row = []
-                    for goal in goal_names:
-                        similarity = vectorizer.calculate_similarity(
-                            doc_vectors[file], goal_vectors[goal]
-                        )
-                        row.append(similarity)
-                    goal_doc_matrix.append(row)
-
-                df_goals = pd.DataFrame(
-                    goal_doc_matrix, columns=goal_names, index=files
-                )
-                st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn"))
-
-    # Keep existing similarity search functionality
-    elif search_button:
-        vectorizer = DocumentVectorizer()
-        with st.spinner("Searching similar documents..."):
-            query_vector = vectorizer.get_embedding(search_text)
-            if query_vector:
-                similar_docs = vectorizer.vector_search(query_vector, search_limit)
-
-                if similar_docs:
-                    st.markdown("### Similar Documents Found")
-
-                    # Create DataFrame with numeric similarities
-                    df = pd.DataFrame(similar_docs)
-
-                    # Apply gradient to numeric column
-                    styled_df = df[["name", "similarity"]].style.background_gradient(
-                        cmap="RdYlGn", subset=["similarity"]
-                    )
-
-                    # Format display after styling
-                    styled_df = styled_df.format({"similarity": "{:.1%}"})
-
-                    st.dataframe(styled_df)
-
-                    # Show document contents
-                    for doc in similar_docs:
-                        with st.expander(
-                            f"📄 {doc['name']} (Similarity: {doc['similarity_display']})"
-                        ):
-                            st.text(
-                                doc["text"][:20] + "..."
-                                if len(doc["text"]) > 20
-                                else doc["text"]
-                            )
-                else:
-                    st.info("No similar documents found")
-            else:
-                st.error("Could not process search query")
-
-
-def main():
-    st.title("Multi-Goal Document Analysis")
-
-    with st.sidebar:
-        st.markdown("### Input Section")
-        tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
-        # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
-
-        with tab1:
-            # Multiple goals input
-            num_goals = st.number_input("Number of goals:", min_value=1, value=1)
-            goals = []
-            for i in range(num_goals):
-                goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100)
-                if goal:
-                    goals.append(goal)
-
-            uploaded_files = st.file_uploader(
-                "Upload documents",
-                accept_multiple_files=True,
-                type=["txt", "pdf", "docx"],
-            )
-            analyze_button = (
-                st.button("Analyze Documents") if goals and uploaded_files else None
-            )
-
-        with tab2:
-            # Keep existing similarity search tab
-            search_text = st.text_area("Enter text to find similar documents:")
-            search_limit = st.slider("Number of results", 1, 10, 5)
-            search_button = st.button("Search Similar") if search_text else None
-
-    if analyze_button:
-        analyzer = GoalAnalyzer()
-        vectorizer = DocumentVectorizer()
-
-        # Store vectors
-        doc_vectors = {}
-        goal_vectors = {}
-
-        # Process goals first
-        with st.spinner("Processing goals..."):
-            for i, goal in enumerate(goals):
-                vector = vectorizer.get_embedding(goal)
-                if vector:
-                    goal_vectors[f"Goal {i+1}"] = vector
-                    vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal)
-
-        # Process documents
-        with st.spinner("Processing documents..."):
-            for file in uploaded_files:
-                st.markdown(f"### Analysis for {file.name}")
-
-                if vectorizer.vector_exists(file.name):
-                    st.info(f"Vector already exists for {file.name}")
-                    existing_doc = vectorizer.vectors_collection.find_one(
-                        {"name": file.name}
-                    )
-                    doc_vectors[file.name] = existing_doc["vector"]
-                else:
-                    text = analyzer.extract_text_from_file(file)
-                    if not text:
-                        st.warning(f"Could not extract text from {file.name}")
-                        continue
-
-                    vector = vectorizer.get_embedding(text)
-                    if vector:
-                        doc_vectors[file.name] = vector
-                        vectorizer.store_vector(file.name, vector, text)
-
-                # Display goal similarities
-                st.subheader("Goal Relevance Scores")
-                col1, col2 = st.columns([1, 2])
-
-                with col1:
-                    for goal_name, goal_vector in goal_vectors.items():
-                        similarity = (
-                            vectorizer.calculate_similarity(
-                                doc_vectors[file.name], goal_vector
-                            )
-                            * 100
-                        )
-                        st.metric(f"{goal_name}", f"{similarity:.1f}%")
-
-                with col2:
-                    # Get analysis for all goals combined
-                    analysis = asyncio.run(
-                        analyzer.get_perplexity_analysis(text, " | ".join(goals))
-                    )
-                    display_analysis_results(analysis)
-
-                st.divider()
-
-            # Document similarity matrix
-            if len(doc_vectors) > 1:
-                st.markdown("### Document Similarity Matrix")
-                files = list(doc_vectors.keys())
-                similarity_matrix = []
-
-                for file1 in files:
-                    row = []
-                    for file2 in files:
-                        similarity = vectorizer.calculate_similarity(
-                            doc_vectors[file1], doc_vectors[file2]
-                        )
-                        row.append(similarity)
-                    similarity_matrix.append(row)
-
-                df = pd.DataFrame(similarity_matrix, columns=files, index=files)
-                st.dataframe(df.style.background_gradient(cmap="RdYlGn"))
-
-                # Add goal-document similarity matrix
-                st.markdown("### Goal-Document Similarity Matrix")
-                goal_doc_matrix = []
-                goal_names = list(goal_vectors.keys())
-
-                for file in files:
-                    row = []
-                    for goal in goal_names:
-                        similarity = vectorizer.calculate_similarity(
-                            doc_vectors[file], goal_vectors[goal]
-                        )
-                        row.append(similarity)
-                    goal_doc_matrix.append(row)
-
-                df_goals = pd.DataFrame(
-                    goal_doc_matrix, columns=goal_names, index=files
-                )
-                st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn"))
-
-    # Keep existing similarity search functionality
-    elif search_button:
-        vectorizer = DocumentVectorizer()
-        with st.spinner("Searching similar documents..."):
-            query_vector = vectorizer.get_embedding(search_text)
-            if query_vector:
-                similar_docs = vectorizer.vector_search(query_vector, search_limit)
-
-                if similar_docs:
-                    st.markdown("### Similar Documents Found")
-
-                    # Create DataFrame with numeric similarities
-                    df = pd.DataFrame(similar_docs)
-
-                    # Apply gradient to numeric column
-                    styled_df = df[["name", "similarity"]].style.background_gradient(
-                        cmap="RdYlGn", subset=["similarity"]
-                    )
-
-                    # Format display after styling
-                    styled_df = styled_df.format({"similarity": "{:.1%}"})
-
-                    st.dataframe(styled_df)
-
-                    # Show document contents
-                    for doc in similar_docs:
-                        with st.expander(
-                            f"📄 {doc['name']} (Similarity: {doc['similarity_display']})"
-                        ):
-                            st.text(
-                                doc["text"][:20] + "..."
-                                if len(doc["text"]) > 20
-                                else doc["text"]
-                            )
-                else:
-                    st.info("No similar documents found")
-            else:
-                st.error("Could not process search query")
-
-
-if __name__ == "__main__":
-    main()
+import streamlit as st
+from typing import List, Dict
+import httpx
+from pathlib import Path
+import os
+from dotenv import load_dotenv
+import json
+import numpy as np
+from pymongo import MongoClient
+from openai import OpenAI
+from datetime import datetime
+import asyncio
+import pandas as pd
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_KEY")
+MONGODB_URI = os.getenv("MONGO_URI")
+OPENAI_API_KEY = os.getenv("OPENAI_KEY")
+
+# Initialize MongoDB client
+client = MongoClient(MONGODB_URI)
+db = client["document_analysis"]
+vectors_collection = db["document_vectors"]
+
+# Initialize OpenAI client
+openai_client = OpenAI(api_key=OPENAI_API_KEY)
+
+
+class GoalAnalyzer:
+    def __init__(self):
+        self.api_key = PERPLEXITY_API_KEY
+        self.base_url = "https://api.perplexity.ai/chat/completions"
+
+    def clean_json_string(self, content: str) -> str:
+        """Clean and extract valid JSON from string"""
+        # Remove markdown formatting
+        if "```json" in content:
+            content = content.split("```json")[1].split("```")[0]
+        elif "```" in content:
+            content = content.split("```")[1]
+
+        # Find the JSON object boundaries
+        start_idx = content.find("{")
+        end_idx = content.rfind("}") + 1
+
+        if start_idx != -1 and end_idx > 0:
+            content = content[start_idx:end_idx]
+
+        # Clean up common issues
+        content = content.strip()
+        content = content.replace("\n", "")
+        content = content.replace("'", '"')
+
+        return content
+
+    async def get_perplexity_analysis(self, text: str, goal: str) -> Dict:
+        """Get analysis from Perplexity API"""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        prompt = f"""
+        Analyze the following text in context of the goal: {goal}
+        
+        Text: {text}
+        
+        Provide analysis in the following JSON format:
+        {{
+            "themes": ["theme1", "theme2"],
+            "subthemes": {{"theme1": ["subtheme1", "subtheme2"], "theme2": ["subtheme3"]}},
+            "keywords": ["keyword1", "keyword2"],
+            "relevance_score": 0-100
+        }}
+        """
+
+        try:
+            async with httpx.AsyncClient() as client:
+                payload = {
+                    "model": "llama-3.1-sonar-small-128k-chat",  # Updated to supported model
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": "You are an AI assistant that analyzes documents and provides structured analysis.",
+                        },
+                        {"role": "user", "content": prompt},
+                    ],
+                    "max_tokens": 1024,
+                }
+
+                # Debug info using expander
+                with st.expander("Debug Info", expanded=False):
+                    st.write("Request payload:", payload)
+
+                response = await client.post(
+                    self.base_url, headers=headers, json=payload, timeout=30.0
+                )
+
+                # Debug response info
+                with st.expander("Response Info", expanded=False):
+                    st.write("Response status:", response.status_code)
+                    st.write("Response headers:", dict(response.headers))
+                    st.write("Response content:", response.text)
+
+                if response.status_code != 200:
+                    error_detail = (
+                        response.json() if response.content else "No error details"
+                    )
+                    raise Exception(
+                        f"API returned status code {response.status_code}. Details: {error_detail}"
+                    )
+
+                result = response.json()
+                content = (
+                    result.get("choices", [{}])[0].get("message", {}).get("content", "")
+                )
+
+                # Clean and parse JSON
+                cleaned_content = self.clean_json_string(content)
+
+                try:
+                    analysis = json.loads(cleaned_content)
+
+                    # Validate required fields
+                    required_fields = [
+                        "themes",
+                        "subthemes",
+                        "keywords",
+                        "relevance_score",
+                    ]
+                    for field in required_fields:
+                        if field not in analysis:
+                            analysis[field] = [] if field != "relevance_score" else 0
+
+                    return analysis
+
+                except json.JSONDecodeError as e:
+                    st.error(f"JSON parsing error: {str(e)}")
+                    st.error(f"Failed content: {cleaned_content}")
+                    return {
+                        "themes": ["Error parsing themes"],
+                        "subthemes": {"Error": ["Failed to parse subthemes"]},
+                        "keywords": ["parsing-error"],
+                        "relevance_score": 0,
+                    }
+
+        except Exception as e:
+            st.error(f"API Error: {str(e)}")
+            return None
+
+    def extract_text_from_file(self, file) -> str:
+        """Extract text content from uploaded file"""
+        try:
+            text = ""
+            file_type = file.type
+
+            if file_type == "text/plain":
+                text = file.getvalue().decode("utf-8")
+            elif file_type == "application/pdf":
+                import PyPDF2
+
+                pdf_reader = PyPDF2.PdfReader(file)
+                for page in pdf_reader.pages:
+                    text += page.extract_text()
+            elif (
+                file_type
+                == "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+            ):
+                import docx
+
+                doc = docx.Document(file)
+                text = " ".join([paragraph.text for paragraph in doc.paragraphs])
+
+            return text
+        except Exception as e:
+            st.error(f"Error extracting text: {str(e)}")
+            return ""
+
+
+class DocumentVectorizer:
+    def __init__(self):
+        self.model = "text-embedding-ada-002"
+        self.client = MongoClient(MONGODB_URI)
+        self.db = self.client["document_analysis"]
+        self.vectors_collection = self.db["document_vectors"]
+
+        # Create vector search index if it doesn't exist
+        try:
+            self.vectors_collection.create_index(
+                [("vector", "2dsphere")],  # Changed to 2dsphere for vector indexing
+                {
+                    "vectorSearchConfig": {
+                        "dimensions": 1536,  # OpenAI embedding dimensions
+                        "similarity": "cosine",
+                    }
+                },
+            )
+        except Exception as e:
+            st.warning(f"Vector index may already exist")
+
+    def get_embedding(self, text: str) -> list:
+        """Get embedding vector for text using OpenAI"""
+        try:
+            response = openai_client.embeddings.create(model=self.model, input=text)
+            return response.data[0].embedding
+        except Exception as e:
+            st.error(f"Error getting embedding: {str(e)}")
+            return None
+
+    # Add this method to DocumentVectorizer class
+    def vector_exists(self, doc_name: str) -> bool:
+        """Check if vector exists for document"""
+        return self.vectors_collection.count_documents({"name": doc_name}) > 0
+
+    # Update store_vector method in DocumentVectorizer class
+    def store_vector(self, doc_name: str, vector: list, text: str, goal: str = None):
+        """Store document/goal vector in MongoDB using upsert"""
+        try:
+            vector_doc = {
+                "name": doc_name,
+                "vector": vector,
+                "text": text,
+                "type": "document" if goal is None else "goal",
+                "goal": goal,
+                "updated_at": datetime.utcnow(),
+            }
+
+            # Use update_one with upsert
+            self.vectors_collection.update_one(
+                {"name": doc_name},
+                {"$set": vector_doc, "$setOnInsert": {"created_at": datetime.utcnow()}},
+                upsert=True,
+            )
+
+        except Exception as e:
+            st.error(f"Error storing vector: {str(e)}")
+
+    # Update vector_search method in DocumentVectorizer class
+    def vector_search(self, query_vector: List[float], limit: int = 5) -> List[Dict]:
+        """Search for similar documents using vector similarity"""
+        try:
+            # Get all documents
+            documents = list(self.vectors_collection.find({"type": "document"}))
+
+            # Calculate similarities
+            similarities = []
+            for doc in documents:
+                similarity = self.calculate_similarity(query_vector, doc["vector"])
+                similarities.append(
+                    {
+                        "name": doc["name"],
+                        "text": doc["text"],
+                        "similarity": similarity,  # Keep as float
+                        "similarity_display": f"{similarity*100:.1f}%",  # Add display version
+                    }
+                )
+
+            # Sort by similarity and get top k
+            sorted_docs = sorted(
+                similarities,
+                key=lambda x: x["similarity"],  # Sort by float value
+                reverse=True,
+            )[:limit]
+
+            return sorted_docs
+
+        except Exception as e:
+            st.error(f"Vector search error: {str(e)}")
+            return []
+
+    def find_similar_documents(self, text: str, limit: int = 5) -> List[Dict]:
+        """Find similar documents for given text"""
+        vector = self.get_embedding(text)
+        if vector:
+            return self.vector_search(vector, limit)
+        return []
+
+    def calculate_similarity(self, vector1: list, vector2: list) -> float:
+        """Calculate cosine similarity between two vectors"""
+        return np.dot(vector1, vector2) / (
+            np.linalg.norm(vector1) * np.linalg.norm(vector2)
+        )
+
+
+def display_analysis_results(analysis: Dict):
+    """Display analysis results in Streamlit UI"""
+    if not analysis:
+        return
+
+    # Display Themes
+    st.subheader("Themes")
+    for theme in analysis.get("themes", []):
+        with st.expander(f"🎯 {theme}"):
+            # Display subthemes for this theme
+            subthemes = analysis.get("subthemes", {}).get(theme, [])
+            if subthemes:
+                st.write("**Subthemes:**")
+                for subtheme in subthemes:
+                    st.write(f"- {subtheme}")
+
+    # Display Keywords
+    st.subheader("Keywords")
+    keywords = analysis.get("keywords", [])
+    st.write(" | ".join([f"🔑 {keyword}" for keyword in keywords]))
+
+    # Display Relevance Score
+    score = analysis.get("relevance_score", 0)
+    st.metric("Relevance Score", f"{score}%")
+
+
+def display_analyst_dashboard():
+    st.title("Multi-Goal Document Analysis")
+
+    with st.sidebar:
+        st.markdown("### Input Section")
+        tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
+        # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
+
+        with tab1:
+            # Multiple goals input
+            num_goals = st.number_input("Number of goals:", min_value=1, value=1)
+            goals = []
+            for i in range(num_goals):
+                goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100)
+                if goal:
+                    goals.append(goal)
+
+            uploaded_files = st.file_uploader(
+                "Upload documents",
+                accept_multiple_files=True,
+                type=["txt", "pdf", "docx"],
+            )
+            analyze_button = (
+                st.button("Analyze Documents") if goals and uploaded_files else None
+            )
+
+        with tab2:
+            # Keep existing similarity search tab
+            search_text = st.text_area("Enter text to find similar documents:")
+            search_limit = st.slider("Number of results", 1, 10, 5)
+            search_button = st.button("Search Similar") if search_text else None
+
+        if st.button("Logout", use_container_width=True):
+            for key in st.session_state.keys():
+                del st.session_state[key]
+            st.rerun()
+
+    if analyze_button:
+        analyzer = GoalAnalyzer()
+        vectorizer = DocumentVectorizer()
+
+        # Store vectors
+        doc_vectors = {}
+        goal_vectors = {}
+
+        # Process goals first
+        with st.spinner("Processing goals..."):
+            for i, goal in enumerate(goals):
+                vector = vectorizer.get_embedding(goal)
+                if vector:
+                    goal_vectors[f"Goal {i+1}"] = vector
+                    vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal)
+
+        # Process documents
+        with st.spinner("Processing documents..."):
+            for file in uploaded_files:
+                st.markdown(f"### Analysis for {file.name}")
+
+                if vectorizer.vector_exists(file.name):
+                    st.info(f"Vector already exists for {file.name}")
+                    existing_doc = vectorizer.vectors_collection.find_one(
+                        {"name": file.name}
+                    )
+                    doc_vectors[file.name] = existing_doc["vector"]
+                else:
+                    text = analyzer.extract_text_from_file(file)
+                    if not text:
+                        st.warning(f"Could not extract text from {file.name}")
+                        continue
+
+                    vector = vectorizer.get_embedding(text)
+                    if vector:
+                        doc_vectors[file.name] = vector
+                        vectorizer.store_vector(file.name, vector, text)
+
+                # Display goal similarities
+                st.subheader("Goal Relevance Scores")
+                col1, col2 = st.columns([1, 2])
+
+                with col1:
+                    for goal_name, goal_vector in goal_vectors.items():
+                        similarity = (
+                            vectorizer.calculate_similarity(
+                                doc_vectors[file.name], goal_vector
+                            )
+                            * 100
+                        )
+                        st.metric(f"{goal_name}", f"{similarity:.1f}%")
+
+                with col2:
+                    # Get analysis for all goals combined
+                    analysis = asyncio.run(
+                        analyzer.get_perplexity_analysis(text, " | ".join(goals))
+                    )
+                    display_analysis_results(analysis)
+
+                st.divider()
+
+            # Document similarity matrix
+            if len(doc_vectors) > 1:
+                st.markdown("### Document Similarity Matrix")
+                files = list(doc_vectors.keys())
+                similarity_matrix = []
+
+                for file1 in files:
+                    row = []
+                    for file2 in files:
+                        similarity = vectorizer.calculate_similarity(
+                            doc_vectors[file1], doc_vectors[file2]
+                        )
+                        row.append(similarity)
+                    similarity_matrix.append(row)
+
+                df = pd.DataFrame(similarity_matrix, columns=files, index=files)
+                st.dataframe(df.style.background_gradient(cmap="RdYlGn"))
+
+                # Add goal-document similarity matrix
+                st.markdown("### Goal-Document Similarity Matrix")
+                goal_doc_matrix = []
+                goal_names = list(goal_vectors.keys())
+
+                for file in files:
+                    row = []
+                    for goal in goal_names:
+                        similarity = vectorizer.calculate_similarity(
+                            doc_vectors[file], goal_vectors[goal]
+                        )
+                        row.append(similarity)
+                    goal_doc_matrix.append(row)
+
+                df_goals = pd.DataFrame(
+                    goal_doc_matrix, columns=goal_names, index=files
+                )
+                st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn"))
+
+    # Keep existing similarity search functionality
+    elif search_button:
+        vectorizer = DocumentVectorizer()
+        with st.spinner("Searching similar documents..."):
+            query_vector = vectorizer.get_embedding(search_text)
+            if query_vector:
+                similar_docs = vectorizer.vector_search(query_vector, search_limit)
+
+                if similar_docs:
+                    st.markdown("### Similar Documents Found")
+
+                    # Create DataFrame with numeric similarities
+                    df = pd.DataFrame(similar_docs)
+
+                    # Apply gradient to numeric column
+                    styled_df = df[["name", "similarity"]].style.background_gradient(
+                        cmap="RdYlGn", subset=["similarity"]
+                    )
+
+                    # Format display after styling
+                    styled_df = styled_df.format({"similarity": "{:.1%}"})
+
+                    st.dataframe(styled_df)
+
+                    # Show document contents
+                    for doc in similar_docs:
+                        with st.expander(
+                            f"📄 {doc['name']} (Similarity: {doc['similarity_display']})"
+                        ):
+                            st.text(
+                                doc["text"][:20] + "..."
+                                if len(doc["text"]) > 20
+                                else doc["text"]
+                            )
+                else:
+                    st.info("No similar documents found")
+            else:
+                st.error("Could not process search query")
+
+
+def main():
+    st.title("Multi-Goal Document Analysis")
+
+    with st.sidebar:
+        st.markdown("### Input Section")
+        tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
+        # tab1, tab2 = st.tabs(["Document Analysis", "Similarity Search"])
+
+        with tab1:
+            # Multiple goals input
+            num_goals = st.number_input("Number of goals:", min_value=1, value=1)
+            goals = []
+            for i in range(num_goals):
+                goal = st.text_area(f"Goal {i+1}:", key=f"goal_{i}", height=100)
+                if goal:
+                    goals.append(goal)
+
+            uploaded_files = st.file_uploader(
+                "Upload documents",
+                accept_multiple_files=True,
+                type=["txt", "pdf", "docx"],
+            )
+            analyze_button = (
+                st.button("Analyze Documents") if goals and uploaded_files else None
+            )
+
+        with tab2:
+            # Keep existing similarity search tab
+            search_text = st.text_area("Enter text to find similar documents:")
+            search_limit = st.slider("Number of results", 1, 10, 5)
+            search_button = st.button("Search Similar") if search_text else None
+
+    if analyze_button:
+        analyzer = GoalAnalyzer()
+        vectorizer = DocumentVectorizer()
+
+        # Store vectors
+        doc_vectors = {}
+        goal_vectors = {}
+
+        # Process goals first
+        with st.spinner("Processing goals..."):
+            for i, goal in enumerate(goals):
+                vector = vectorizer.get_embedding(goal)
+                if vector:
+                    goal_vectors[f"Goal {i+1}"] = vector
+                    vectorizer.store_vector(f"Goal {i+1}", vector, goal, goal)
+
+        # Process documents
+        with st.spinner("Processing documents..."):
+            for file in uploaded_files:
+                st.markdown(f"### Analysis for {file.name}")
+
+                if vectorizer.vector_exists(file.name):
+                    st.info(f"Vector already exists for {file.name}")
+                    existing_doc = vectorizer.vectors_collection.find_one(
+                        {"name": file.name}
+                    )
+                    doc_vectors[file.name] = existing_doc["vector"]
+                else:
+                    text = analyzer.extract_text_from_file(file)
+                    if not text:
+                        st.warning(f"Could not extract text from {file.name}")
+                        continue
+
+                    vector = vectorizer.get_embedding(text)
+                    if vector:
+                        doc_vectors[file.name] = vector
+                        vectorizer.store_vector(file.name, vector, text)
+
+                # Display goal similarities
+                st.subheader("Goal Relevance Scores")
+                col1, col2 = st.columns([1, 2])
+
+                with col1:
+                    for goal_name, goal_vector in goal_vectors.items():
+                        similarity = (
+                            vectorizer.calculate_similarity(
+                                doc_vectors[file.name], goal_vector
+                            )
+                            * 100
+                        )
+                        st.metric(f"{goal_name}", f"{similarity:.1f}%")
+
+                with col2:
+                    # Get analysis for all goals combined
+                    analysis = asyncio.run(
+                        analyzer.get_perplexity_analysis(text, " | ".join(goals))
+                    )
+                    display_analysis_results(analysis)
+
+                st.divider()
+
+            # Document similarity matrix
+            if len(doc_vectors) > 1:
+                st.markdown("### Document Similarity Matrix")
+                files = list(doc_vectors.keys())
+                similarity_matrix = []
+
+                for file1 in files:
+                    row = []
+                    for file2 in files:
+                        similarity = vectorizer.calculate_similarity(
+                            doc_vectors[file1], doc_vectors[file2]
+                        )
+                        row.append(similarity)
+                    similarity_matrix.append(row)
+
+                df = pd.DataFrame(similarity_matrix, columns=files, index=files)
+                st.dataframe(df.style.background_gradient(cmap="RdYlGn"))
+
+                # Add goal-document similarity matrix
+                st.markdown("### Goal-Document Similarity Matrix")
+                goal_doc_matrix = []
+                goal_names = list(goal_vectors.keys())
+
+                for file in files:
+                    row = []
+                    for goal in goal_names:
+                        similarity = vectorizer.calculate_similarity(
+                            doc_vectors[file], goal_vectors[goal]
+                        )
+                        row.append(similarity)
+                    goal_doc_matrix.append(row)
+
+                df_goals = pd.DataFrame(
+                    goal_doc_matrix, columns=goal_names, index=files
+                )
+                st.dataframe(df_goals.style.background_gradient(cmap="RdYlGn"))
+
+    # Keep existing similarity search functionality
+    elif search_button:
+        vectorizer = DocumentVectorizer()
+        with st.spinner("Searching similar documents..."):
+            query_vector = vectorizer.get_embedding(search_text)
+            if query_vector:
+                similar_docs = vectorizer.vector_search(query_vector, search_limit)
+
+                if similar_docs:
+                    st.markdown("### Similar Documents Found")
+
+                    # Create DataFrame with numeric similarities
+                    df = pd.DataFrame(similar_docs)
+
+                    # Apply gradient to numeric column
+                    styled_df = df[["name", "similarity"]].style.background_gradient(
+                        cmap="RdYlGn", subset=["similarity"]
+                    )
+
+                    # Format display after styling
+                    styled_df = styled_df.format({"similarity": "{:.1%}"})
+
+                    st.dataframe(styled_df)
+
+                    # Show document contents
+                    for doc in similar_docs:
+                        with st.expander(
+                            f"📄 {doc['name']} (Similarity: {doc['similarity_display']})"
+                        ):
+                            st.text(
+                                doc["text"][:20] + "..."
+                                if len(doc["text"]) > 20
+                                else doc["text"]
+                            )
+                else:
+                    st.info("No similar documents found")
+            else:
+                st.error("Could not process search query")
+
+
+if __name__ == "__main__":
+    main()

infranew.py

@@ -0,0 +1,231 @@
+import streamlit as st
+import pandas as pd
+import networkx as nx
+from bokeh.models import HoverTool
+from bokeh.plotting import figure, from_networkx
+import requests
+import json
+import google.generativeai as genai
+
+PERPLEXITY_API_KEY = "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"
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+
+
+def extract_edges(keywords):
+    keywords = [kw.strip() for kw in keywords.split(",")]
+    edges = [
+        (keywords[i], keywords[j])
+        for i in range(len(keywords))
+        for j in range(i + 1, len(keywords))
+    ]
+    return edges
+
+
+def create_knowledge_graph(data):
+    G = nx.Graph()
+
+    for _, row in data.iterrows():
+        words = []
+        for col in data.columns:
+            if pd.notnull(row[col]):
+                # Convert to string and handle numeric values
+                cell_value = str(row[col]).strip()
+                if cell_value:
+                    words.extend(cell_value.split())
+
+        if words:
+            edges = extract_edges(",".join(words))
+            G.add_edges_from(edges)
+
+            for word in words:
+                word = word.strip()
+                if word not in G:
+                    G.add_node(word, title=word, value=len(word))
+
+    return G
+
+
+def render_graph_bokeh(G):
+    plot = figure(
+        title="Interactive Knowledge Graph",
+        x_range=(-1.5, 1.5),
+        y_range=(-1.5, 1.5),
+        tools="pan,wheel_zoom,box_zoom,reset,tap",
+        active_scroll="wheel_zoom",
+    )
+    plot.add_tools(HoverTool(tooltips="@index"))
+
+    graph_renderer = from_networkx(G, nx.spring_layout, scale=1, center=(0, 0))
+
+    graph_renderer.node_renderer.glyph.size = 10
+    graph_renderer.node_renderer.glyph.fill_color = "blue"
+    graph_renderer.node_renderer.glyph.line_color = "black"
+
+    graph_renderer.edge_renderer.glyph.line_width = 1
+    graph_renderer.edge_renderer.glyph.line_color = "gray"
+
+    plot.renderers.append(graph_renderer)
+
+    return plot
+
+
+import re
+
+
+def search_papers(topic: str, num_papers: int) -> list:
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    prompt = f"""Find {num_papers} recent research papers about {topic}.
+    Return ONLY a valid JSON array with the following structure for each paper:
+    [
+        {{
+            "Title": "paper title",
+            "Abstract": "abstract text",
+            "Keywords": "key terms"
+        }}
+    ]"""
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are a research paper analyzer that returns valid JSON arrays.",
+            },
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.1,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+
+        # Clean response to ensure valid JSON
+        content = content.strip()
+        if not content.startswith("["):
+            content = content[content.find("[") :]
+        if not content.endswith("]"):
+            content = content[: content.rfind("]") + 1]
+
+        # Remove any trailing commas before closing brackets
+        content = re.sub(r",\s*]", "]", content)
+        content = re.sub(r",\s*}", "}", content)
+
+        papers = json.loads(content)
+        if not isinstance(papers, list):
+            raise ValueError("Response is not a JSON array")
+        return papers
+    except requests.exceptions.RequestException as e:
+        st.error(f"API Request Error: {str(e)}")
+        return []
+    except json.JSONDecodeError as e:
+        st.error(f"Invalid JSON response: {str(e)}")
+        st.error(f"Response content: {response.text}")
+        return []
+    except ValueError as e:
+        st.error(f"Error: {str(e)}")
+        return []
+
+
+import os
+
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+GEMINI_API_URL = "https://api.openai.com/v1/engines/davinci-codex/completions"
+
+
+def call_gemini_api(prompt: str) -> str:
+    headers = {
+        "Authorization": f"Bearer {GEMINI_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "prompt": prompt,
+        "max_tokens": 150,
+        "temperature": 0.7,
+    }
+
+    try:
+        model = genai.GenerativeModel("gemini-pro")
+        response = model.generate_content(prompt)
+        return response.text
+    except Exception as e:
+        st.error(f"Gemini API Error: {str(e)}")
+        return ""
+
+
+def generate_gaps_paragraph(gaps):
+    prompt = f"Generate a brief paragraph about the gaps in the research based on the following gaps: {', '.join(gaps)}"
+    return call_gemini_api(prompt)
+
+
+def generate_insights(G, topic):
+    papers = search_papers(topic, 5)
+    if papers:
+        st.write("### Research Insights from Perplexity API")
+        for paper in papers:
+            st.write(f"**Title:** {paper['Title']}")
+            st.write(f"**Abstract:** {paper['Abstract']}")
+            st.write(f"**Keywords:** {paper['Keywords']}")
+            st.write("---")
+
+    nodes = list(G.nodes(data=True))
+    insights = {}
+    insights["Strong Points"] = [
+        n for n, d in nodes if G.degree(n) > len(G.nodes) * 0.1
+    ]
+    insights["Weak Points"] = [n for n, d in nodes if G.degree(n) < len(G.nodes) * 0.05]
+    insights["Gaps"] = [n for n, d in nodes if len(list(nx.neighbors(G, n))) == 0]
+
+    st.write("### Graph-Based Insights")
+    st.write("**Strong Points:**", insights["Strong Points"])
+    st.write("**Weak Points:**", insights["Weak Points"])
+    st.write("**Gaps:**", insights["Gaps"])
+
+    if insights["Gaps"]:
+        with st.spinner("Generating insights about gaps..."):
+            gaps_paragraph = generate_gaps_paragraph(insights["Gaps"])
+            if gaps_paragraph:
+                st.write("### Gaps in Research")
+                st.write(gaps_paragraph)
+
+
+def main():
+    st.title("Advanced Interactive Knowledge Graph")
+    st.write(
+        "Upload a CSV file to generate a fully interactive and insightful knowledge graph."
+    )
+
+    uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
+
+    if uploaded_file is not None:
+        try:
+            data = pd.read_csv(uploaded_file)
+            st.write("Preview of the uploaded data:")
+            st.dataframe(data.head())
+
+            G = create_knowledge_graph(data)
+
+            st.write("Generated Knowledge Graph:")
+            plot = render_graph_bokeh(G)
+            st.bokeh_chart(plot, use_container_width=True)
+
+            topic = st.text_input(
+                "Enter a topic for additional insights:", "knowledge graphs"
+            )
+            if topic:
+                generate_insights(G, topic)
+
+        except Exception as e:
+            st.error(f"An error occurred while processing the file: {e}")
+    else:
+        st.info("Please upload a CSV file to get started.")
+
+
+if __name__ == "__main__":
+    main()

keywords_database_download.py

@@ -0,0 +1,104 @@
+import streamlit as st
+import pandas as pd
+from pymongo import MongoClient
+from dotenv import load_dotenv
+import os
+import json
+import re
+
+# 1. Load environment variables
+load_dotenv()
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+# 2. Create MongoDB connection
+client = MongoClient(MONGODB_URI)
+db = client["novascholar_db"]
+collection = db["research_papers"]
+
+
+def convert_mixed_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Convert any columns that contain lists into comma-separated strings
+    in order to ensure consistent data types for CSV export.
+    """
+    for col in df.columns:
+        if any(isinstance(val, list) for val in df[col].dropna()):
+            df[col] = df[col].apply(
+                lambda x: (
+                    ", ".join(map(str, x))
+                    if isinstance(x, list)
+                    else (str(x) if pd.notna(x) else "")
+                )
+            )
+    return df
+
+
+def filter_and_export_collection_to_csv(keyword: str, doc_collection=None):
+    """
+    Find documents in the given collection with a matching keyword
+    in the 'Keywords' field, export them to CSV, and return the DataFrame
+    and CSV filename.
+    """
+    # Use the default 'research_papers' collection if none provided
+    if doc_collection is None:
+        doc_collection = collection
+
+    docs = list(doc_collection.find({"Keywords": {"$regex": keyword, "$options": "i"}}))
+    if docs:
+        df = pd.DataFrame(docs)
+        df = convert_mixed_columns(df)
+        csv_filename = "papers_filtered_export.csv"
+        df.to_csv(csv_filename, index=False)
+        return df, csv_filename
+    else:
+        # Return an empty DataFrame if no documents found
+        return pd.DataFrame(), None
+
+
+def main():
+    # st.set_page_config(page_title="Filter and Export Papers", layout="wide")
+    st.title("Filter and Export Papers by Keyword")
+
+    # Let user select the paper type
+    paper_type = st.selectbox(
+        "Select type of research paper:",
+        [
+            "Review Based Paper",
+            "Opinion/Perspective Based Paper",
+            "Empirical Research Paper",
+            "Research Paper (Other)",
+        ],
+    )
+
+    # 5. Let user enter the keyword to filter
+    keyword_input = st.text_input(
+        "Enter the exact keyword to filter papers by 'Keywords' field:"
+    )
+
+    # When user clicks button, use the collection for the selected paper type
+    if st.button("Export Filtered Papers to CSV"):
+        with st.spinner("Exporting filtered documents..."):
+            try:
+                # Determine dynamic collection based on paper type
+                collection_name = paper_type.replace(" ", "_").lower()
+                doc_collection = db[collection_name]
+
+                df, csv_filename = filter_and_export_collection_to_csv(
+                    keyword_input, doc_collection
+                )
+                if not df.empty and csv_filename:
+                    st.success(
+                        f"Successfully exported filtered papers to {csv_filename}!"
+                    )
+                    st.write("Preview of the filtered DataFrame:")
+                    st.dataframe(df)
+                else:
+                    st.warning("No matching documents found for that keyword.")
+            except Exception as e:
+                st.error(f"Error exporting filtered papers: {str(e)}")
+
+
+if __name__ == "__main__":
+    main()

live_polls.py

@@ -1,115 +1,115 @@
-# live_poll_feature.py
-
-import streamlit as st
-import pandas as pd
-from datetime import datetime
-from poll_db_operations import PollDatabase
-
-class LivePollFeature:
-    def __init__(self):
-        self.db = PollDatabase()
-    
-    def display_faculty_interface(self, session_id):
-        """Display the faculty interface for managing polls"""
-        st.subheader("Live Polls Management")
-        
-        # Create new poll
-        with st.expander("Create New Poll", expanded=False):
-            question = st.text_input("Poll Question")
-            
-            num_options = st.number_input("Number of Options", 
-                                        min_value=2, 
-                                        max_value=6, 
-                                        value=4)
-            
-            options = []
-            for i in range(num_options):
-                option = st.text_input(f"Option {i+1}", 
-                                     key=f"option_{i}")
-                if option:
-                    options.append(option)
-            
-            if st.button("Create Poll") and question and len(options) >= 2:
-                self.db.create_poll(
-                    st.session_state.selected_course,
-                    session_id,
-                    question,
-                    options,
-                    st.session_state.user_id
-                )
-                st.success("Poll created successfully!")
-                st.rerun()
-        
-        # Display active polls
-        active_polls = self.db.get_active_polls(session_id)
-        if active_polls:
-            st.subheader("Active Polls")
-            for poll in active_polls:
-                with st.expander(f"Poll: {poll['question']}", expanded=True):
-                    # Display results
-                    self._display_poll_results(poll)
-                    
-                    if st.button("Close Poll", 
-                               key=f"close_{str(poll['_id'])}"):
-                        self.db.close_poll(poll['_id'])
-                        st.success("Poll closed successfully!")
-                        st.rerun()
-    
-    def display_student_interface(self, session_id):
-        """Display the student interface for participating in polls"""
-        st.subheader("Live Polls")
-        
-        active_polls = self.db.get_active_polls(session_id)
-        if not active_polls:
-            st.info("No active polls at the moment.")
-            return
-        
-        for poll in active_polls:
-            with st.expander(f"Poll: {poll['question']}", expanded=True):
-                selected_option = st.radio(
-                    "Your response:",
-                    options=poll['options'],
-                    key=f"poll_{str(poll['_id'])}"
-                )
-                
-                if st.button("Submit Response", 
-                           key=f"submit_{str(poll['_id'])}"):
-                    success, message = self.db.submit_response(
-                        poll['_id'],
-                        st.session_state.user_id,
-                        selected_option
-                    )
-                    if success:
-                        st.success(message)
-                    else:
-                        st.warning(message)
-                    st.rerun()
-                
-                # self._display_poll_results(poll)
-    
-    def _display_poll_results(self, poll):
-        """Helper method to display poll results"""
-        responses_df = pd.DataFrame(
-            list(poll['responses'].items()),
-            columns=['Option', 'Votes']
-        )
-        
-        total_votes = responses_df['Votes'].sum()
-        
-        # Calculate percentages
-        if total_votes > 0:
-            responses_df['Percentage'] = (
-                responses_df['Votes'] / total_votes * 100
-            ).round(1)
-        else:
-            responses_df['Percentage'] = 0
-        
-        # Display metrics
-        st.metric("Total Responses", total_votes)
-        
-        # Display charts
-        st.bar_chart(responses_df.set_index('Option')['Votes'])
-        
-        # Display detailed statistics
-        if st.session_state.user_type == 'faculty':
+# live_poll_feature.py
+
+import streamlit as st
+import pandas as pd
+from datetime import datetime
+from poll_db_operations import PollDatabase
+
+class LivePollFeature:
+    def __init__(self):
+        self.db = PollDatabase()
+    
+    def display_faculty_interface(self, session_id):
+        """Display the faculty interface for managing polls"""
+        st.subheader("Live Polls Management")
+        
+        # Create new poll
+        with st.expander("Create New Poll", expanded=False):
+            question = st.text_input("Poll Question")
+            
+            num_options = st.number_input("Number of Options", 
+                                        min_value=2, 
+                                        max_value=6, 
+                                        value=4)
+            
+            options = []
+            for i in range(num_options):
+                option = st.text_input(f"Option {i+1}", 
+                                     key=f"option_{i}")
+                if option:
+                    options.append(option)
+            
+            if st.button("Create Poll") and question and len(options) >= 2:
+                self.db.create_poll(
+                    st.session_state.selected_course,
+                    session_id,
+                    question,
+                    options,
+                    st.session_state.user_id
+                )
+                st.success("Poll created successfully!")
+                st.rerun()
+        
+        # Display active polls
+        active_polls = self.db.get_active_polls(session_id)
+        if active_polls:
+            st.subheader("Active Polls")
+            for poll in active_polls:
+                with st.expander(f"Poll: {poll['question']}", expanded=True):
+                    # Display results
+                    self._display_poll_results(poll)
+                    
+                    if st.button("Close Poll", 
+                               key=f"close_{str(poll['_id'])}"):
+                        self.db.close_poll(poll['_id'])
+                        st.success("Poll closed successfully!")
+                        st.rerun()
+    
+    def display_student_interface(self, session_id):
+        """Display the student interface for participating in polls"""
+        st.subheader("Live Polls")
+        
+        active_polls = self.db.get_active_polls(session_id)
+        if not active_polls:
+            st.info("No active polls at the moment.")
+            return
+        
+        for poll in active_polls:
+            with st.expander(f"Poll: {poll['question']}", expanded=True):
+                selected_option = st.radio(
+                    "Your response:",
+                    options=poll['options'],
+                    key=f"poll_{str(poll['_id'])}"
+                )
+                
+                if st.button("Submit Response", 
+                           key=f"submit_{str(poll['_id'])}"):
+                    success, message = self.db.submit_response(
+                        poll['_id'],
+                        st.session_state.user_id,
+                        selected_option
+                    )
+                    if success:
+                        st.success(message)
+                    else:
+                        st.warning(message)
+                    st.rerun()
+                
+                # self._display_poll_results(poll)
+    
+    def _display_poll_results(self, poll):
+        """Helper method to display poll results"""
+        responses_df = pd.DataFrame(
+            list(poll['responses'].items()),
+            columns=['Option', 'Votes']
+        )
+        
+        total_votes = responses_df['Votes'].sum()
+        
+        # Calculate percentages
+        if total_votes > 0:
+            responses_df['Percentage'] = (
+                responses_df['Votes'] / total_votes * 100
+            ).round(1)
+        else:
+            responses_df['Percentage'] = 0
+        
+        # Display metrics
+        st.metric("Total Responses", total_votes)
+        
+        # Display charts
+        st.bar_chart(responses_df.set_index('Option')['Votes'])
+        
+        # Display detailed statistics
+        if st.session_state.user_type == 'faculty':
             st.dataframe(responses_df)

loldude.py

@@ -0,0 +1,135 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import plotly.express as px
+import plotly.graph_objects as go
+from collections import defaultdict
+
+def load_and_preprocess_data(uploaded_file):
+    """Load and preprocess the CSV data."""
+    df = pd.read_csv(uploaded_file)
+    # Combine relevant text fields for similarity comparison
+    df['combined_text'] = df['Title'] + ' ' + df['Abstract'] + ' ' + df['Keywords']
+    return df
+
+def calculate_similarity_matrix(df):
+    """Calculate cosine similarity matrix based on combined text."""
+    tfidf = TfidfVectorizer(stop_words='english')
+    tfidf_matrix = tfidf.fit_transform(df['combined_text'])
+    similarity_matrix = cosine_similarity(tfidf_matrix)
+    return similarity_matrix
+
+def find_similar_papers(similarity_matrix, df, threshold=0.7):
+    """Find pairs of papers with similarity above threshold."""
+    similar_pairs = []
+    for i in range(len(similarity_matrix)):
+        for j in range(i + 1, len(similarity_matrix)):
+            similarity = similarity_matrix[i][j]
+            if similarity >= threshold:
+                similar_pairs.append({
+                    'Paper 1': df.iloc[i]['Title'],
+                    'Paper 2': df.iloc[j]['Title'],
+                    'Similarity': similarity
+                })
+    return pd.DataFrame(similar_pairs)
+
+def find_outliers(similarity_matrix, df, threshold=0.3):
+    """Find papers with low average similarity to others."""
+    avg_similarities = np.mean(similarity_matrix, axis=1)
+    outliers = []
+    for i, avg_sim in enumerate(avg_similarities):
+        if avg_sim < threshold:
+            outliers.append({
+                'Title': df.iloc[i]['Title'],
+                'Average Similarity': avg_sim
+            })
+    return pd.DataFrame(outliers)
+
+def create_similarity_heatmap(similarity_matrix, df):
+    """Create a heatmap of similarity matrix."""
+    fig = go.Figure(data=go.Heatmap(
+        z=similarity_matrix,
+        x=df['Title'],
+        y=df['Title'],
+        colorscale='Viridis'
+    ))
+    fig.update_layout(
+        title='Paper Similarity Heatmap',
+        xaxis_tickangle=-45,
+        height=800
+    )
+    return fig
+
+def analyze_keywords(df):
+    """Analyze keyword frequency across papers."""
+    keyword_freq = defaultdict(int)
+    for keywords in df['Keywords']:
+        if isinstance(keywords, str):
+            for keyword in keywords.split(','):
+                keyword = keyword.strip()
+                keyword_freq[keyword] += 1
+    
+    keyword_df = pd.DataFrame([
+        {'Keyword': k, 'Frequency': v} 
+        for k, v in keyword_freq.items()
+    ]).sort_values('Frequency', ascending=False)
+    
+    return keyword_df
+
+def main():
+    st.title('Research Papers Similarity Analysis')
+    
+    uploaded_file = st.file_uploader("Upload your research papers CSV file", type=['csv'])
+    
+    if uploaded_file is not None:
+        df = load_and_preprocess_data(uploaded_file)
+        similarity_matrix = calculate_similarity_matrix(df)
+        
+        st.header('Document Similarity Analysis')
+        
+        # Similarity Heatmap
+        st.subheader('Similarity Heatmap')
+        heatmap = create_similarity_heatmap(similarity_matrix, df)
+        st.plotly_chart(heatmap, use_container_width=True)
+        
+        # Similar Papers
+        st.subheader('Similar Papers')
+        similarity_threshold = st.slider('Similarity Threshold', 0.0, 1.0, 0.7)
+        similar_papers = find_similar_papers(similarity_matrix, df, similarity_threshold)
+        if not similar_papers.empty:
+            st.dataframe(similar_papers)
+        else:
+            st.write("No papers found above the similarity threshold.")
+        
+        # Outliers
+        st.subheader('Outlier Papers')
+        outlier_threshold = st.slider('Outlier Threshold', 0.0, 1.0, 0.3)
+        outliers = find_outliers(similarity_matrix, df, outlier_threshold)
+        if not outliers.empty:
+            st.dataframe(outliers)
+        else:
+            st.write("No outliers found below the threshold.")
+        
+        # Keyword Analysis
+        st.header('Keyword Analysis')
+        keyword_freq = analyze_keywords(df)
+        if not keyword_freq.empty:
+            fig = px.bar(keyword_freq, x='Keyword', y='Frequency',
+                        title='Keyword Frequency Across Papers')
+            fig.update_xaxes(tickangle=45)
+            st.plotly_chart(fig, use_container_width=True)
+        
+        # Basic Statistics
+        st.header('Basic Statistics')
+        col1, col2 = st.columns(2)
+        with col1:
+            st.metric("Total Papers", len(df))
+            st.metric("Average Similarity", f"{np.mean(similarity_matrix):.2f}")
+        with col2:
+            st.metric("Unique Keywords", len(keyword_freq))
+            st.metric("Max Similarity", f"{np.max(similarity_matrix[~np.eye(similarity_matrix.shape[0], dtype=bool)]):.2f}")
+
+if __name__ == "__main__":
+    main()

modify_schema.py

@@ -1,222 +1,222 @@
-from db import courses_collection2
-from dotenv import load_dotenv
-import os
-from pymongo import MongoClient
-from datetime import datetime
-
-
-
-load_dotenv()
-MONGO_URI = os.getenv("MONGO_URI")
-
-client = MongoClient(MONGO_URI)
-db = client["novascholar_db"]
-
-# Define the updated course schema
-updated_course_schema = {
-    "bsonType": "object",
-    "required": [
-        "course_id",
-        "title",
-        "description",
-        "faculty",
-        "faculty_id",
-        "duration",
-        "created_at",
-    ],
-    "properties": {
-        "course_id": {
-            "bsonType": "string",
-            "description": "Unique identifier for the course",
-        },
-        "title": {"bsonType": "string", "description": "Title of the course"},
-        "description": {
-            "bsonType": "string",
-            "description": "Description of the course",
-        },
-        "faculty": {"bsonType": "string", "description": "Name of the faculty"},
-        "duration": {"bsonType": "string", "description": "Duration of the course"},
-        "created_at": {
-            "bsonType": "date",
-            "description": "Date when the course was created",
-        },
-        "sessions": {
-            "bsonType": "array",
-            "description": "List of sessions associated with the course",
-            "items": {
-                "bsonType": "object",
-                "required": ["session_id", "title", "date"],
-                "properties": {
-                    "session_id": {
-                        "bsonType": "string",
-                        "description": "Unique identifier for the session",
-                    },
-                    "title": {
-                        "bsonType": "string",
-                        "description": "Title of the session",
-                    },
-                    "date": {"bsonType": "date", "description": "Date of the session"},
-                    "status": {
-                        "bsonType": "string",
-                        "description": "Status of the session (e.g., completed, upcoming)",
-                    },
-                    "created_at": {
-                        "bsonType": "date",
-                        "description": "Date when the session was created",
-                    },
-                    "pre_class": {
-                        "bsonType": "object",
-                        "description": "Pre-class segment data",
-                        "properties": {
-                            "resources": {
-                                "bsonType": "array",
-                                "description": "List of pre-class resources",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["type", "title", "url"],
-                                    "properties": {
-                                        "type": {
-                                            "bsonType": "string",
-                                            "description": "Type of resource (e.g., pdf, video)",
-                                        },
-                                        "title": {
-                                            "bsonType": "string",
-                                            "description": "Title of the resource",
-                                        },
-                                        "url": {
-                                            "bsonType": "string",
-                                            "description": "URL of the resource",
-                                        },
-                                        "vector": {
-                                            "bsonType": "array",
-                                            "description": "Vector representation of the resource",
-                                            "items": {"bsonType": "double"},
-                                        },
-                                    },
-                                },
-                            },
-                            "completion_required": {
-                                "bsonType": "bool",
-                                "description": "Indicates if completion of pre-class resources is required",
-                            },
-                        },
-                    },
-                    "in_class": {
-                        "bsonType": "object",
-                        "description": "In-class segment data",
-                        "properties": {
-                            "topics": {
-                                "bsonType": "array",
-                                "description": "List of topics covered in the session",
-                                "items": {"bsonType": "string"},
-                            },
-                            "quiz": {
-                                "bsonType": "object",
-                                "description": "Quiz data",
-                                "properties": {
-                                    "title": {
-                                        "bsonType": "string",
-                                        "description": "Title of the quiz",
-                                    },
-                                    "questions": {
-                                        "bsonType": "int",
-                                        "description": "Number of questions in the quiz",
-                                    },
-                                    "duration": {
-                                        "bsonType": "int",
-                                        "description": "Duration of the quiz in minutes",
-                                    },
-                                },
-                            },
-                            "polls": {
-                                "bsonType": "array",
-                                "description": "List of polls conducted during the session",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["question", "options"],
-                                    "properties": {
-                                        "question": {
-                                            "bsonType": "string",
-                                            "description": "Poll question",
-                                        },
-                                        "options": {
-                                            "bsonType": "array",
-                                            "description": "List of poll options",
-                                            "items": {"bsonType": "string"},
-                                        },
-                                        "responses": {
-                                            "bsonType": "object",
-                                            "description": "Responses to the poll",
-                                            "additionalProperties": {"bsonType": "int"},
-                                        },
-                                    },
-                                },
-                            },
-                        },
-                    },
-                    "post_class": {
-                        "bsonType": "object",
-                        "description": "Post-class segment data",
-                        "properties": {
-                            "assignments": {
-                                "bsonType": "array",
-                                "description": "List of assignments",
-                                "items": {
-                                    "bsonType": "object",
-                                    "required": ["id", "title", "due_date", "status"],
-                                    "properties": {
-                                        "id": {
-                                            "bsonType": ["objectId", "int"],
-                                            "description": "Assignment ID",
-                                        },
-                                        "title": {
-                                            "bsonType": "string",
-                                            "description": "Title of the assignment",
-                                        },
-                                        "due_date": {
-                                            "bsonType": "date",
-                                            "description": "Due date of the assignment",
-                                        },
-                                        "status": {
-                                            "bsonType": "string",
-                                            "description": "Status of the assignment (e.g., pending, completed)",
-                                        },
-                                        "submissions": {
-                                            "bsonType": "array",
-                                            "description": "List of submissions",
-                                            "items": {
-                                                "bsonType": "object",
-                                                "properties": {
-                                                    "student_id": {
-                                                        "bsonType": "objectId",
-                                                        "description": "ID of the student who submitted the assignment",
-                                                    },
-                                                    "file_url": {
-                                                        "bsonType": "string",
-                                                        "description": "URL of the submitted file",
-                                                    },
-                                                    "submitted_at": {
-                                                        "bsonType": "date",
-                                                        "description": "Date when the assignment was submitted",
-                                                    },
-                                                },
-                                            },
-                                        },
-                                    },
-                                },
-                            }
-                        },
-                    },
-                },
-            },
-        },
-    },
-}
-
-# Update the schema using the collMod command
-db.command({
-    "collMod": "courses_collection2",
-    "validator": {"$jsonSchema": updated_course_schema}
-})
-
+from db import courses_collection2
+from dotenv import load_dotenv
+import os
+from pymongo import MongoClient
+from datetime import datetime
+
+
+
+load_dotenv()
+MONGO_URI = os.getenv("MONGO_URI")
+
+client = MongoClient(MONGO_URI)
+db = client["novascholar_db"]
+
+# Define the updated course schema
+updated_course_schema = {
+    "bsonType": "object",
+    "required": [
+        "course_id",
+        "title",
+        "description",
+        "faculty",
+        "faculty_id",
+        "duration",
+        "created_at",
+    ],
+    "properties": {
+        "course_id": {
+            "bsonType": "string",
+            "description": "Unique identifier for the course",
+        },
+        "title": {"bsonType": "string", "description": "Title of the course"},
+        "description": {
+            "bsonType": "string",
+            "description": "Description of the course",
+        },
+        "faculty": {"bsonType": "string", "description": "Name of the faculty"},
+        "duration": {"bsonType": "string", "description": "Duration of the course"},
+        "created_at": {
+            "bsonType": "date",
+            "description": "Date when the course was created",
+        },
+        "sessions": {
+            "bsonType": "array",
+            "description": "List of sessions associated with the course",
+            "items": {
+                "bsonType": "object",
+                "required": ["session_id", "title", "date"],
+                "properties": {
+                    "session_id": {
+                        "bsonType": "string",
+                        "description": "Unique identifier for the session",
+                    },
+                    "title": {
+                        "bsonType": "string",
+                        "description": "Title of the session",
+                    },
+                    "date": {"bsonType": "date", "description": "Date of the session"},
+                    "status": {
+                        "bsonType": "string",
+                        "description": "Status of the session (e.g., completed, upcoming)",
+                    },
+                    "created_at": {
+                        "bsonType": "date",
+                        "description": "Date when the session was created",
+                    },
+                    "pre_class": {
+                        "bsonType": "object",
+                        "description": "Pre-class segment data",
+                        "properties": {
+                            "resources": {
+                                "bsonType": "array",
+                                "description": "List of pre-class resources",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["type", "title", "url"],
+                                    "properties": {
+                                        "type": {
+                                            "bsonType": "string",
+                                            "description": "Type of resource (e.g., pdf, video)",
+                                        },
+                                        "title": {
+                                            "bsonType": "string",
+                                            "description": "Title of the resource",
+                                        },
+                                        "url": {
+                                            "bsonType": "string",
+                                            "description": "URL of the resource",
+                                        },
+                                        "vector": {
+                                            "bsonType": "array",
+                                            "description": "Vector representation of the resource",
+                                            "items": {"bsonType": "double"},
+                                        },
+                                    },
+                                },
+                            },
+                            "completion_required": {
+                                "bsonType": "bool",
+                                "description": "Indicates if completion of pre-class resources is required",
+                            },
+                        },
+                    },
+                    "in_class": {
+                        "bsonType": "object",
+                        "description": "In-class segment data",
+                        "properties": {
+                            "topics": {
+                                "bsonType": "array",
+                                "description": "List of topics covered in the session",
+                                "items": {"bsonType": "string"},
+                            },
+                            "quiz": {
+                                "bsonType": "object",
+                                "description": "Quiz data",
+                                "properties": {
+                                    "title": {
+                                        "bsonType": "string",
+                                        "description": "Title of the quiz",
+                                    },
+                                    "questions": {
+                                        "bsonType": "int",
+                                        "description": "Number of questions in the quiz",
+                                    },
+                                    "duration": {
+                                        "bsonType": "int",
+                                        "description": "Duration of the quiz in minutes",
+                                    },
+                                },
+                            },
+                            "polls": {
+                                "bsonType": "array",
+                                "description": "List of polls conducted during the session",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["question", "options"],
+                                    "properties": {
+                                        "question": {
+                                            "bsonType": "string",
+                                            "description": "Poll question",
+                                        },
+                                        "options": {
+                                            "bsonType": "array",
+                                            "description": "List of poll options",
+                                            "items": {"bsonType": "string"},
+                                        },
+                                        "responses": {
+                                            "bsonType": "object",
+                                            "description": "Responses to the poll",
+                                            "additionalProperties": {"bsonType": "int"},
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    "post_class": {
+                        "bsonType": "object",
+                        "description": "Post-class segment data",
+                        "properties": {
+                            "assignments": {
+                                "bsonType": "array",
+                                "description": "List of assignments",
+                                "items": {
+                                    "bsonType": "object",
+                                    "required": ["id", "title", "due_date", "status"],
+                                    "properties": {
+                                        "id": {
+                                            "bsonType": ["objectId", "int"],
+                                            "description": "Assignment ID",
+                                        },
+                                        "title": {
+                                            "bsonType": "string",
+                                            "description": "Title of the assignment",
+                                        },
+                                        "due_date": {
+                                            "bsonType": "date",
+                                            "description": "Due date of the assignment",
+                                        },
+                                        "status": {
+                                            "bsonType": "string",
+                                            "description": "Status of the assignment (e.g., pending, completed)",
+                                        },
+                                        "submissions": {
+                                            "bsonType": "array",
+                                            "description": "List of submissions",
+                                            "items": {
+                                                "bsonType": "object",
+                                                "properties": {
+                                                    "student_id": {
+                                                        "bsonType": "objectId",
+                                                        "description": "ID of the student who submitted the assignment",
+                                                    },
+                                                    "file_url": {
+                                                        "bsonType": "string",
+                                                        "description": "URL of the submitted file",
+                                                    },
+                                                    "submitted_at": {
+                                                        "bsonType": "date",
+                                                        "description": "Date when the assignment was submitted",
+                                                    },
+                                                },
+                                            },
+                                        },
+                                    },
+                                },
+                            }
+                        },
+                    },
+                },
+            },
+        },
+    },
+}
+
+# Update the schema using the collMod command
+db.command({
+    "collMod": "courses_collection2",
+    "validator": {"$jsonSchema": updated_course_schema}
+})
+
 print("Schema updated successfully!")

new_keywords.py

@@ -0,0 +1,127 @@
+import streamlit as st
+import pandas as pd
+from pymongo import MongoClient
+from dotenv import load_dotenv
+import os
+import json
+import re
+
+# 1. Load environment variables
+load_dotenv()
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+# 2. Create MongoDB connection
+client = MongoClient(MONGODB_URI)
+db = client["novascholar_db"]
+collection = db["research_papers"]
+
+
+def convert_mixed_columns(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Convert any columns that contain lists into comma-separated strings
+    to ensure consistent data types for CSV export.
+    """
+    for col in df.columns:
+        if any(isinstance(val, list) for val in df[col].dropna()):
+            df[col] = df[col].apply(
+                lambda x: (
+                    ", ".join(map(str, x))
+                    if isinstance(x, list)
+                    else (str(x) if pd.notna(x) else "")
+                )
+            )
+    return df
+
+
+def filter_and_export_collection_to_csv(keywords_list, doc_collection):
+    """
+    Fetch documents from the specified collection where the 'Keywords' field
+    matches ANY of the keywords in 'keywords_list'. Convert to DataFrame,
+    ensure consistent column types, save to CSV, and return the DataFrame
+    and CSV filename.
+    """
+    # 3. Retrieve filtered documents from the collection based on 'Keywords' using $in with regex for substring matching
+    regex_keywords = [f".*{keyword}.*" for keyword in keywords_list]
+    docs = list(
+        doc_collection.find(
+            {"Keywords": {"$regex": "|".join(regex_keywords), "$options": "i"}}
+        )
+    )
+
+    # Convert documents to DataFrame
+    df = pd.DataFrame(docs)
+
+    if not df.empty:
+        # 4. Convert mixed columns
+        df = convert_mixed_columns(df)
+        # 5. Export to CSV
+        csv_filename = "filtered_papers_export.csv"
+        df.to_csv(csv_filename, index=False)
+        return df, csv_filename
+    else:
+        # Return an empty DataFrame and None if no documents found
+        return pd.DataFrame(), None
+
+
+def main():
+    st.title("Filter and Export Papers by Keyword")
+
+    # Let user select the paper type
+    paper_type = st.selectbox(
+        "Select type of research paper:",
+        [
+            "Review Based Paper",
+            "Opinion/Perspective Based Paper",
+            "Empirical Research Paper",
+            "Research Paper (Other)",
+        ],
+    )
+
+    # Let user enter the keyword to filter
+    keyword_input = st.text_input(
+        "Enter the exact keyword to filter papers by 'Keywords' field:"
+    )
+
+    # When user clicks button, use the collection for the selected paper type
+    if st.button("Export Filtered Papers to CSV"):
+        with st.spinner("Exporting filtered documents..."):
+            try:
+                # Determine dynamic collection based on paper type
+                collection_name = paper_type.replace(" ", "_").lower()
+                doc_collection = db[collection_name]
+
+                # Split keywords by commas and strip whitespace
+                keywords_list = [
+                    kw.strip() for kw in keyword_input.split(",") if kw.strip()
+                ]
+
+                if not keywords_list:
+                    st.warning("Please enter at least one keyword.")
+                else:
+                    df, csv_filename = filter_and_export_collection_to_csv(
+                        keywords_list, doc_collection
+                    )
+                    if not df.empty and csv_filename:
+                        st.success(
+                            f"Successfully exported filtered papers to {csv_filename}!"
+                        )
+                        st.download_button(
+                            label="Download CSV",
+                            data=df.to_csv(index=False).encode("utf-8"),
+                            file_name=csv_filename,
+                            mime="text/csv",
+                        )
+                        st.write("Preview of the filtered DataFrame:")
+                        st.dataframe(df)
+                    else:
+                        st.warning(
+                            "No matching documents found for the provided keyword(s)."
+                        )
+            except Exception as e:
+                st.error(f"Error exporting filtered papers: {str(e)}")
+
+
+if __name__ == "__main__":
+    main()

new_research_paper.py

@@ -0,0 +1,103 @@
+import streamlit as st
+import pandas as pd
+import requests
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+
+
+def call_perplexity_api(prompt: str) -> str:
+    """Call Perplexity AI with a prompt, return the text response if successful."""
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.3,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        st.error(f"API Error: {str(e)}")
+        return ""
+
+
+def generate_research_paper(df: pd.DataFrame) -> dict:
+    """
+    For each column in the DataFrame, generate a research paper section (200-500 words)
+    that addresses the data in that column. Return a dict mapping column -> text.
+    """
+    paper_sections = {}
+    for col in df.columns:
+        # Convert all non-null rows in the column to strings and join them for context
+        col_values = df[col].dropna().astype(str).tolist()
+        # We'll truncate if this is huge
+        sample_text = " | ".join(col_values[:50])  # limit to first 50 rows for brevity
+        prompt = f"""
+        Topic: {col}
+        Data Sample: {sample_text}
+
+        Generate a professional research paper section for the above column.
+        The section should be at least 100 words and at most 150 words,
+        focusing on key insights, challenges, and potential research angles.
+        Integrate the data samples as context for the content.
+        """
+        section_text = call_perplexity_api(prompt)
+        paper_sections[col] = section_text.strip() if section_text else ""
+    return paper_sections
+
+
+def format_paper(paper_dict: dict) -> str:
+    """
+    Format the generated paper into a Markdown string.
+    Each column name is used as a heading, and the text is placed under it.
+    """
+    md_text = "# Generated Research Paper\n\n"
+    for col, content in paper_dict.items():
+        md_text += f"## {col}\n{content}\n\n"
+    return md_text
+
+
+def main():
+    st.title("Corpus-based Research Paper Generator")
+
+    uploaded_file = st.file_uploader("Upload CSV corpus file", type="csv")
+    if uploaded_file:
+        df = pd.read_csv(uploaded_file)
+        st.write("### Preview of Uploaded Data")
+        st.dataframe(df.head())
+
+        if st.button("Generate Research Paper"):
+            st.info("Generating paper based on the columns of your corpus...")
+            with st.spinner("Calling Perplexity AI..."):
+                paper = generate_research_paper(df)
+                if paper:
+                    formatted_paper = format_paper(paper)
+                    st.success("Research Paper Generated Successfully!")
+                    st.write(formatted_paper)
+
+                    st.download_button(
+                        label="Download Paper as Markdown",
+                        data=formatted_paper,
+                        file_name="research_paper.md",
+                        mime="text/markdown",
+                    )
+                else:
+                    st.error(
+                        "Paper generation failed. Please check Perplexity API key."
+                    )
+
+
+if __name__ == "__main__":
+    main()

poll_db_operations.py

@@ -1,70 +1,70 @@
-from pymongo import MongoClient
-from datetime import datetime
-from bson import ObjectId
-from dotenv import load_dotenv
-import os
-
-load_dotenv()
-MONGO_URI = os.getenv('MONGO_URI')
-class PollDatabase:
-    def __init__(self):
-        self.client = MongoClient(MONGO_URI)
-        self.db = self.client["novascholar_db"]
-    
-    def create_poll(self, course_id, session_id, question, options, faculty_id):
-        """Create a new poll"""
-        poll = {
-            "course_id": course_id,
-            "session_id": session_id,
-            "faculty_id": faculty_id,
-            "question": question,
-            "options": options,
-            "status": "active",
-            "created_at": datetime.now(),
-            "responses": {option: 0 for option in options}
-        }
-        return self.db.polls.insert_one(poll)
-    
-    def get_active_polls(self, session_id):
-        """Get all active polls for a session"""
-        return list(self.db.polls.find({
-            "session_id": session_id,
-            "status": "active"
-        }))
-    
-    def submit_response(self, poll_id, student_id, selected_option):
-        """Submit a student's response to a poll"""
-        try:
-            # Record individual response
-            response = {
-                "poll_id": poll_id,
-                "student_id": student_id,
-                "selected_option": selected_option,
-                "submitted_at": datetime.now()
-            }
-            self.db.poll_responses.insert_one(response)
-            
-            # Update aggregated results
-            self.db.polls.update_one(
-                {"_id": ObjectId(poll_id)},
-                {"$inc": {f"responses.{selected_option}": 1}}
-            )
-            return True, "Vote recorded successfully"
-            
-        except Exception as e:
-            if "duplicate key error" in str(e):
-                return False, "You have already voted in this poll"
-            return False, f"Error recording vote: {str(e)}"
-    
-    def close_poll(self, poll_id):
-        """Close a poll"""
-        return self.db.polls.update_one(
-            {"_id": ObjectId(poll_id)},
-            {"$set": {"status": "closed"}}
-        )
-    
-    def get_poll_analytics(self, poll_id):
-        """Get detailed analytics for a poll"""
-        poll = self.db.polls.find_one({"_id": ObjectId(poll_id)})
-        responses = self.db.poll_responses.find({"poll_id": ObjectId(poll_id)})
+from pymongo import MongoClient
+from datetime import datetime
+from bson import ObjectId
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+class PollDatabase:
+    def __init__(self):
+        self.client = MongoClient(MONGO_URI)
+        self.db = self.client["novascholar_db"]
+    
+    def create_poll(self, course_id, session_id, question, options, faculty_id):
+        """Create a new poll"""
+        poll = {
+            "course_id": course_id,
+            "session_id": session_id,
+            "faculty_id": faculty_id,
+            "question": question,
+            "options": options,
+            "status": "active",
+            "created_at": datetime.now(),
+            "responses": {option: 0 for option in options}
+        }
+        return self.db.polls.insert_one(poll)
+    
+    def get_active_polls(self, session_id):
+        """Get all active polls for a session"""
+        return list(self.db.polls.find({
+            "session_id": session_id,
+            "status": "active"
+        }))
+    
+    def submit_response(self, poll_id, student_id, selected_option):
+        """Submit a student's response to a poll"""
+        try:
+            # Record individual response
+            response = {
+                "poll_id": poll_id,
+                "student_id": student_id,
+                "selected_option": selected_option,
+                "submitted_at": datetime.now()
+            }
+            self.db.poll_responses.insert_one(response)
+            
+            # Update aggregated results
+            self.db.polls.update_one(
+                {"_id": ObjectId(poll_id)},
+                {"$inc": {f"responses.{selected_option}": 1}}
+            )
+            return True, "Vote recorded successfully"
+            
+        except Exception as e:
+            if "duplicate key error" in str(e):
+                return False, "You have already voted in this poll"
+            return False, f"Error recording vote: {str(e)}"
+    
+    def close_poll(self, poll_id):
+        """Close a poll"""
+        return self.db.polls.update_one(
+            {"_id": ObjectId(poll_id)},
+            {"$set": {"status": "closed"}}
+        )
+    
+    def get_poll_analytics(self, poll_id):
+        """Get detailed analytics for a poll"""
+        poll = self.db.polls.find_one({"_id": ObjectId(poll_id)})
+        responses = self.db.poll_responses.find({"poll_id": ObjectId(poll_id)})
         return poll, list(responses)

poll_db_setup.py

@@ -1,35 +1,35 @@
-from pymongo import MongoClient
-from datetime import datetime
-from dotenv import load_dotenv
-import os
-
-load_dotenv()
-MONGO_URI = os.getenv('MONGO_URI')
-def setup_mongodb():
-    """Initialize MongoDB connection and create collections with indexes"""
-    client = MongoClient(MONGO_URI)
-    db = client["novascholar_db"]
-    
-    # Create indexes for polls collection
-    db.polls.create_index([("session_id", 1), ("status", 1)])
-    db.polls.create_index([("course_id", 1)])
-    
-    # Create unique index for poll_responses to prevent duplicate votes
-    db.poll_responses.create_index(
-        [("poll_id", 1), ("student_id", 1)],
-        unique=True
-    )
-    
-    return "Database setup completed successfully"
-
-def print_all_polls():
-    """Print all polls in the database"""
-    client = MongoClient(MONGO_URI)
-    db = client["novascholar_db"]
-    
-    polls = db.polls.find()
-    for poll in polls:
-        print(poll)
-
-if __name__ == "__main__":
+from pymongo import MongoClient
+from datetime import datetime
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+def setup_mongodb():
+    """Initialize MongoDB connection and create collections with indexes"""
+    client = MongoClient(MONGO_URI)
+    db = client["novascholar_db"]
+    
+    # Create indexes for polls collection
+    db.polls.create_index([("session_id", 1), ("status", 1)])
+    db.polls.create_index([("course_id", 1)])
+    
+    # Create unique index for poll_responses to prevent duplicate votes
+    db.poll_responses.create_index(
+        [("poll_id", 1), ("student_id", 1)],
+        unique=True
+    )
+    
+    return "Database setup completed successfully"
+
+def print_all_polls():
+    """Print all polls in the database"""
+    client = MongoClient(MONGO_URI)
+    db = client["novascholar_db"]
+    
+    polls = db.polls.find()
+    for poll in polls:
+        print(poll)
+
+if __name__ == "__main__":
     print(print_all_polls())

pre_class_analytics2.py

@@ -1,759 +1,759 @@
-import json
-import typing_extensions as typing
-import google.generativeai as genai
-from typing import List, Dict, Any
-import numpy as np
-from collections import defaultdict
-
-from dotenv import load_dotenv
-import os
-import pymongo
-from pymongo import MongoClient
-
-load_dotenv()
-GEMINI_API_KEY = os.getenv('GEMINI_KEY')
-
-class EngagementMetrics(typing.TypedDict):
-    participation_level: str  # "high" | "medium" | "low"
-    question_quality: str     # "advanced" | "intermediate" | "basic"
-    concept_understanding: str  # "strong" | "moderate" | "needs_improvement"
-
-class StudentInsight(typing.TypedDict):
-    student_id: str
-    performance_level: str  # "high_performer" | "average" | "at_risk"
-    struggling_topics: list[str]
-    engagement_metrics: EngagementMetrics
-
-class TopicInsight(typing.TypedDict):
-    topic: str
-    difficulty_level: float  # 0 to 1
-    student_count: int
-    common_issues: list[str]
-    key_misconceptions: list[str]
-
-class RecommendedAction(typing.TypedDict):
-    action: str
-    priority: str  # "high" | "medium" | "low"
-    target_group: str  # "all_students" | "specific_students" | "faculty"
-    reasoning: str
-    expected_impact: str
-
-class ClassDistribution(typing.TypedDict):
-    high_performers: float
-    average_performers: float
-    at_risk: float
-
-class CourseHealth(typing.TypedDict):
-    overall_engagement: float  # 0 to 1
-    critical_topics: list[str]
-    class_distribution: ClassDistribution
-
-class InterventionMetrics(typing.TypedDict):
-    immediate_attention_needed: list[str]  # student_ids
-    monitoring_required: list[str]  # student_ids
-
-class AnalyticsResponse(typing.TypedDict):
-    topic_insights: list[TopicInsight]
-    student_insights: list[StudentInsight]
-    recommended_actions: list[RecommendedAction]
-    course_health: CourseHealth
-    intervention_metrics: InterventionMetrics
-
-
-
-class NovaScholarAnalytics:
-    def __init__(self, model_name: str = "gemini-1.5-flash"):
-        genai.configure(api_key=GEMINI_API_KEY)
-        self.model = genai.GenerativeModel(model_name)
-        
-    def _create_analytics_prompt(self, chat_histories: List[Dict], all_topics: List[str]) -> str:
-        """Creates a structured prompt for Gemini to analyze chat histories."""
-        # Prompt 1: 
-        # return f"""Analyze these student chat histories for a university course and provide detailed analytics.
-
-        # Context:
-        # - These are pre-class chat interactions between students and an AI tutor
-        # - Topics covered: {', '.join(all_topics)}
-
-        # Chat histories: {json.dumps(chat_histories, indent=2)}
-
-        # Return the analysis in JSON format matching this exact schema:
-        # {AnalyticsResponse.__annotations__}
-
-        # Ensure all numeric values are between 0 and 1 (accuracy upto 3 decimal places) where applicable.
-
-        # Important analysis guidelines:
-        # 1. Identify topics where students show confusion or ask multiple follow-up questions
-        # 2. Look for patterns in question types and complexity
-        # 3. Analyze response understanding based on follow-up questions
-        # 4. Consider both explicit and implicit signs of difficulty
-        # 5. Focus on concept relationships and prerequisite understanding"""
-
-        # Prompt 2: 
-        # return f"""Analyze the provided student chat histories for a university course and generate concise, actionable analytics.  
-
-        # Context:  
-        # - Chat histories: {json.dumps(chat_histories, indent=2)}
-        # - These are pre-class interactions between students and an AI tutor aimed at identifying learning difficulties and improving course delivery.  
-        # - Topics covered: {', '.join(all_topics)}.  
-
-        # Your task is to extract key insights that will help faculty address challenges effectively and enhance learning outcomes.  
-
-        # Output Format:  
-        # 1. Topics where students face significant difficulties:  
-        # - Provide a ranked list of topics where the majority of students are struggling, based on the frequency and nature of their questions or misconceptions.  
-        # - Include the percentage of students who found each topic challenging.  
-
-        # 2. AI-recommended actions for faculty:  
-        # - Suggest actionable steps to address the difficulties identified in each critical topic.  
-        # - Specify the priority of each action (high, medium, low) based on the urgency and impact.  
-        # - Explain the reasoning behind each recommendation and its expected impact on student outcomes.  
-
-        # 3. Student-specific analytics (focusing on at-risk students):  
-        # - Identify students categorized as "at-risk" based on their engagement levels, question complexity, and recurring struggles.  
-        # - For each at-risk student, list their top 3 struggling topics and their engagement metrics (participation level, concept understanding).  
-        # - Provide personalized recommendations for improving their understanding.  
-
-        # Guidelines for Analysis:  
-        # - Focus on actionable and concise insights rather than exhaustive details.  
-        # - Use both explicit (e.g., direct questions) and implicit (e.g., repeated follow-ups) cues to identify areas of difficulty.  
-        # - Prioritize topics with higher difficulty scores or more students struggling.  
-        # - Ensure numerical values (e.g., difficulty levels, percentages) are between 0 and 1 where applicable.  
-
-        # The response must be well-structured, concise, and highly actionable for faculty to implement improvements effectively."""  
-
-        # Prompt 3:
-        return f"""Analyze the provided student chat histories for a university course and generate concise, actionable analytics.
-        Context:
-        - Chat histories: {json.dumps(chat_histories, indent=2)}
-        - These are pre-class interactions between students and an AI tutor aimed at identifying learning difficulties and improving course delivery.
-        - Topics covered: {', '.join(all_topics)}.
-
-        Your task is to provide detailed analytics that will help faculty address challenges effectively and enhance learning outcomes.
-
-        Output Format (strictly follow this JSON structure):
-        {{
-        "topic_wise_insights": [
-            {{
-            "topic": "<string>",
-            "struggling_percentage": <number between 0 and 1>,
-            "key_issues": ["<string>", "<string>", ...],
-            "key_misconceptions": ["<string>", "<string>", ...],
-            "recommended_actions": {{
-                "description": "<string>",
-                "priority": "high|medium|low",
-                "expected_outcome": "<string>"
-            }}
-            }}
-        ],
-        "ai_recommended_actions": [
-        {{
-            "action": "<string>",
-            "priority": "high|medium|low",
-            "reasoning": "<string>",
-            "expected_outcome": "<string>",
-            "pedagogy_recommendations": {{
-                "methods": ["<string>", "<string>", ...],
-                "resources": ["<string>", "<string>", ...],
-                "expected_impact": "<string>"
-            }}
-        }}
-        ],
-        "student_analytics": [
-            {{
-            "student_id": "<string>",
-            "engagement_metrics": {{
-                "participation_level": <number between 0 and 1>,
-                "concept_understanding": "strong|moderate|needs_improvement",
-                "question_quality": "advanced|intermediate|basic"
-            }},
-            "struggling_topics": ["<string>", "<string>", ...],
-            "personalized_recommendation": "<string>"
-            }}
-        ]
-        }}
-
-        Guidelines for Analysis:
-        - Focus on actionable and concise insights rather than exhaustive details.
-        - Use both explicit (e.g., direct questions) and implicit (e.g., repeated follow-ups) cues to identify areas of difficulty.
-        - Prioritize topics with higher difficulty scores or more students struggling.
-        - Ensure numerical values (e.g., difficulty levels, percentages) are between 0 and 1 where applicable.
-        - Make sure to include All** students in the analysis, not just a subset.
-        - for the ai_recommended_actions:
-            - Prioritize pedagogy recommendations for critical topics with the high difficulty scores or struggling percentages.
-            - For each action:
-                - Include specific teaching methods (e.g., interactive discussions or quizzes, problem-based learning, practical examples etc).
-                - Recommend supporting resources (e.g., videos, handouts, simulations).
-                - Provide reasoning for the recommendation and the expected outcomes for student learning.
-                - Example:
-                - **Action:** Conduct an interactive problem-solving session on "<Topic Name>".
-                - **Reasoning:** Students showed difficulty in applying concepts to practical problems.
-                - **Expected Outcome:** Improved practical understanding and application of the topic.
-                - **Pedagogy Recommendations:**
-                    - **Methods:** Group discussions, real-world case studies.
-                    - **Resources:** Online interactive tools, relevant case studies, video walkthroughs.
-                    - **Expected Impact:** Enhance conceptual clarity by 40% and practical application by 30%. 
-
-        The response must adhere strictly to the above JSON structure, with all fields populated appropriately."""
-
-    
-    def _calculate_class_distribution(self, analytics: Dict) -> Dict:
-        """Calculate the distribution of students across performance levels."""
-        try:
-            total_students = len(analytics.get("student_insights", []))
-            if total_students == 0:
-                return {
-                    "high_performers": 0,
-                    "average_performers": 0,
-                    "at_risk": 0
-                }
-            
-            distribution = defaultdict(int)
-            
-            for student in analytics.get("student_insights", []):
-                performance_level = student.get("performance_level", "average")
-                # Map performance levels to our three categories
-                if performance_level in ["excellent", "high", "high_performer"]:
-                    distribution["high_performers"] += 1
-                elif performance_level in ["struggling", "low", "at_risk"]:
-                    distribution["at_risk"] += 1
-                else:
-                    distribution["average_performers"] += 1
-            
-            # Convert to percentages
-            return {
-                level: count/total_students 
-                for level, count in distribution.items()
-            }
-        except Exception as e:
-            print(f"Error calculating class distribution: {str(e)}")
-            return {
-                "high_performers": 0,
-                "average_performers": 0,
-                "at_risk": 0
-            }
-
-    def _identify_urgent_cases(self, analytics: Dict) -> List[str]:
-        """Identify students needing immediate attention."""
-        try:
-            urgent_cases = []
-            for student in analytics.get("student_insights", []):
-                student_id = student.get("student_id")
-                if not student_id:
-                    continue
-                    
-                # Check multiple risk factors
-                risk_factors = 0
-                
-                # Factor 1: Performance level
-                if student.get("performance_level") in ["struggling", "at_risk", "low"]:
-                    risk_factors += 1
-                    
-                # Factor 2: Number of struggling topics
-                if len(student.get("struggling_topics", [])) >= 2:
-                    risk_factors += 1
-                    
-                # Factor 3: Engagement metrics
-                engagement = student.get("engagement_metrics", {})
-                if (engagement.get("participation_level") == "low" or 
-                    engagement.get("concept_understanding") == "needs_improvement"):
-                    risk_factors += 1
-                
-                # If student has multiple risk factors, add to urgent cases
-                if risk_factors >= 2:
-                    urgent_cases.append(student_id)
-            
-            return urgent_cases
-        except Exception as e:
-            print(f"Error identifying urgent cases: {str(e)}")
-            return []
-
-    def _identify_monitoring_cases(self, analytics: Dict) -> List[str]:
-        """Identify students who need monitoring but aren't urgent cases."""
-        try:
-            monitoring_cases = []
-            urgent_cases = set(self._identify_urgent_cases(analytics))
-            
-            for student in analytics.get("student_insights", []):
-                student_id = student.get("student_id")
-                if not student_id or student_id in urgent_cases:
-                    continue
-                
-                # Check monitoring criteria
-                monitoring_needed = False
-                
-                # Criterion 1: Has some struggling topics but not enough for urgent
-                if len(student.get("struggling_topics", [])) == 1:
-                    monitoring_needed = True
-                    
-                # Criterion 2: Medium-low engagement
-                engagement = student.get("engagement_metrics", {})
-                if engagement.get("participation_level") == "medium":
-                    monitoring_needed = True
-                    
-                # Criterion 3: Recent performance decline
-                if student.get("performance_level") == "average":
-                    monitoring_needed = True
-                
-                if monitoring_needed:
-                    monitoring_cases.append(student_id)
-            
-            return monitoring_cases
-        except Exception as e:
-            print(f"Error identifying monitoring cases: {str(e)}")
-            return []
-
-    def _identify_critical_topics(self, analytics: Dict) -> List[str]:
-        """
-        Identify critical topics that need attention based on multiple factors.
-        Returns a list of topic names that are considered critical.
-        """
-        try:
-            critical_topics = []
-            topics = analytics.get("topic_insights", [])
-            
-            for topic in topics:
-                if not isinstance(topic, dict):
-                    continue
-                    
-                # Initialize score for topic criticality
-                critical_score = 0
-                
-                # Factor 1: High difficulty level
-                difficulty_level = topic.get("difficulty_level", 0)
-                if difficulty_level > 0.7:
-                    critical_score += 2
-                elif difficulty_level > 0.5:
-                    critical_score += 1
-                    
-                # Factor 2: Number of students struggling
-                student_count = topic.get("student_count", 0)
-                total_students = len(analytics.get("student_insights", []))
-                if total_students > 0:
-                    struggle_ratio = student_count / total_students
-                    if struggle_ratio > 0.5:
-                        critical_score += 2
-                    elif struggle_ratio > 0.3:
-                        critical_score += 1
-                        
-                # Factor 3: Number of common issues
-                if len(topic.get("common_issues", [])) > 2:
-                    critical_score += 1
-                    
-                # Factor 4: Number of key misconceptions
-                if len(topic.get("key_misconceptions", [])) > 1:
-                    critical_score += 1
-                
-                # If topic exceeds threshold, mark as critical
-                if critical_score >= 3:
-                    critical_topics.append(topic.get("topic", "Unknown Topic"))
-            
-            return critical_topics
-            
-        except Exception as e:
-            print(f"Error identifying critical topics: {str(e)}")
-            return []
-
-    def _calculate_engagement(self, analytics: Dict) -> Dict:
-        """
-        Calculate detailed engagement metrics across all students.
-        Returns a dictionary with engagement statistics.
-        """
-        try:
-            total_students = len(analytics.get("student_insights", []))
-            if total_students == 0:
-                return {
-                    "total_students": 0,
-                    "overall_score": 0,
-                    "engagement_distribution": {
-                        "high": 0,
-                        "medium": 0,
-                        "low": 0
-                    },
-                    "participation_metrics": {
-                        "average_topics_per_student": 0,
-                        "active_participants": 0
-                    }
-                }
-            
-            engagement_levels = defaultdict(int)
-            total_topics_engaged = 0
-            active_participants = 0
-            
-            for student in analytics.get("student_insights", []):
-                # Get engagement metrics
-                metrics = student.get("engagement_metrics", {})
-                
-                # Calculate participation level
-                participation = metrics.get("participation_level", "low").lower()
-                engagement_levels[participation] += 1
-                
-                # Count topics student is engaged with
-                topics_count = len(student.get("struggling_topics", []))
-                total_topics_engaged += topics_count
-                
-                # Count active participants (students engaging with any topics)
-                if topics_count > 0:
-                    active_participants += 1
-            
-            # Calculate overall engagement score (0-1)
-            weighted_score = (
-                (engagement_levels["high"] * 1.0 + 
-                engagement_levels["medium"] * 0.6 + 
-                engagement_levels["low"] * 0.2) / total_students
-            )
-            
-            return {
-                "total_students": total_students,
-                "overall_score": round(weighted_score, 2),
-                "engagement_distribution": {
-                    level: count/total_students 
-                    for level, count in engagement_levels.items()
-                },
-                "participation_metrics": {
-                    "average_topics_per_student": round(total_topics_engaged / total_students, 2),
-                    "active_participants_ratio": round(active_participants / total_students, 2)
-                }
-            }
-            
-        except Exception as e:
-            print(f"Error calculating engagement: {str(e)}")
-            return {
-                "total_students": 0,
-                "overall_score": 0,
-                "engagement_distribution": {
-                    "high": 0,
-                    "medium": 0,
-                    "low": 0
-                },
-                "participation_metrics": {
-                    "average_topics_per_student": 0,
-                    "active_participants_ratio": 0
-                }
-            }
-
-    def _process_gemini_response(self, response: str) -> Dict:
-        """Process and validate Gemini's response."""
-        # try:
-        #     analytics = json.loads(response)
-        #     return self._enrich_analytics(analytics)
-        # except json.JSONDecodeError as e:
-        #     print(f"Error decoding Gemini response: {e}")
-        #     return self._fallback_analytics()
-        try:
-            # Parse JSON response
-            analytics = json.loads(response)
-            
-            # Validate required fields exist
-            required_fields = {
-                "topic_insights": [],
-                "student_insights": [],
-                "recommended_actions": []
-            }
-            
-            # Ensure all required fields exist with default values
-            for field, default_value in required_fields.items():
-                if field not in analytics or not analytics[field]:
-                    analytics[field] = default_value
-            
-            # Now enrich the validated analytics
-            return self._enrich_analytics(analytics)
-            
-        except (json.JSONDecodeError, KeyError, TypeError) as e:
-            print(f"Error processing Gemini response: {str(e)}")
-            print(f"Raw response: {response}")
-            return self._fallback_analytics()
-
-    def _enrich_analytics(self, analytics: Dict) -> Dict:
-        """Add derived insights and metrics to the analytics."""
-        # Add overall course health metrics
-        analytics["course_health"] = {
-            "overall_engagement": self._calculate_engagement(analytics),
-            "critical_topics": self._identify_critical_topics(analytics),
-            "class_distribution": self._calculate_class_distribution(analytics)
-        }
-        
-        # Add intervention urgency scores
-        analytics["intervention_metrics"] = {
-            "immediate_attention_needed": self._identify_urgent_cases(analytics),
-            "monitoring_required": self._identify_monitoring_cases(analytics)
-        }
-        
-        return analytics
-
-    def _calculate_engagement(self, analytics: Dict) -> Dict:
-        # """Calculate overall engagement metrics."""
-        # total_students = len(analytics["student_insights"])
-        # engagement_levels = defaultdict(int)
-        
-        # for student in analytics["student_insights"]:
-        #     engagement_levels[student["engagement_metrics"]["participation_level"]] += 1
-            
-        # return {
-        #     "total_students": total_students,
-        #     "engagement_distribution": {
-        #         level: count/total_students 
-        #         for level, count in engagement_levels.items()
-        #     }
-        # }
-        """Calculate overall engagement metrics with defensive programming."""
-        try:
-            total_students = len(analytics.get("student_insights", []))
-            if total_students == 0:
-                return {
-                    "total_students": 0,
-                    "engagement_distribution": {
-                        "high": 0,
-                        "medium": 0,
-                        "low": 0
-                    }
-                }
-            
-            engagement_levels = defaultdict(int)
-            
-            for student in analytics.get("student_insights", []):
-                metrics = student.get("engagement_metrics", {})
-                level = metrics.get("participation_level", "low")
-                engagement_levels[level] += 1
-                
-            return {
-                "total_students": total_students,
-                "engagement_distribution": {
-                    level: count/total_students 
-                    for level, count in engagement_levels.items()
-                }
-            }
-        except Exception as e:
-            print(f"Error calculating engagement: {str(e)}")
-            return {
-                "total_students": 0,
-                "engagement_distribution": {
-                    "high": 0,
-                    "medium": 0,
-                    "low": 0
-                }
-            }
-
-    def _identify_critical_topics(self, analytics: Dict) -> List[Dict]:
-        # """Identify topics needing immediate attention."""
-        # return [
-        #     topic for topic in analytics["topic_insights"]
-        #     if topic["difficulty_level"] > 0.7 or 
-        #     len(topic["common_issues"]) > 2
-        # ]
-        """Identify topics needing immediate attention with defensive programming."""
-        try:
-            return [
-                topic for topic in analytics.get("topic_insights", [])
-                if topic.get("difficulty_level", 0) > 0.7 or 
-                len(topic.get("common_issues", [])) > 2
-            ]
-        except Exception as e:
-            print(f"Error identifying critical topics: {str(e)}")
-            return []
-
-    def generate_analytics(self, chat_histories: List[Dict], all_topics: List[str]) -> Dict:
-        # Method 1: (caused key 'student_insights' error): 
-        # """Main method to generate analytics from chat histories."""
-        # # Preprocess chat histories
-        # processed_histories = self._preprocess_chat_histories(chat_histories)
-        
-        # # Create and send prompt to Gemini
-        # prompt = self._create_analytics_prompt(processed_histories, all_topics)
-        # response = self.model.generate_content(
-        #     prompt, 
-        #     generation_config=genai.GenerationConfig(
-        #         response_mime_type="application/json",
-        #         response_schema=AnalyticsResponse
-        #     )
-        # )
-        
-        # # # Process and enrich analytics
-        # # analytics = self._process_gemini_response(response.text)
-        # # return analytics
-        # # Process, validate, and enrich the response
-        # analytics = self._process_gemini_response(response.text)
-    
-        # # Then cast it to satisfy the type checker
-        # return typing.cast(AnalyticsResponse, analytics)
-
-        # Method 2 (possible fix): 
-        # """Main method to generate analytics with better error handling."""
-        # try:
-        #     processed_histories = self._preprocess_chat_histories(chat_histories)
-        #     prompt = self._create_analytics_prompt(processed_histories, all_topics)
-            
-        #     response = self.model.generate_content(
-        #         prompt,
-        #         generation_config=genai.GenerationConfig(
-        #             response_mime_type="application/json",
-        #             temperature=0.15
-        #             # response_schema=AnalyticsResponse
-        #         )
-        #     )
-            
-        #     if not response.text:
-        #         print("Empty response from Gemini")
-        #         return self._fallback_analytics()
-                
-        #     # analytics = self._process_gemini_response(response.text)
-        #     # return typing.cast(AnalyticsResponse, analytics)
-        #     # return response.text;
-        #     analytics = json.loads(response.text)
-        #     return analytics
-        
-        # except Exception as e:
-        #     print(f"Error generating analytics: {str(e)}")
-        #     return self._fallback_analytics()
-
-
-        # Debugging code: 
-        """Main method to generate analytics with better error handling."""
-        try:
-            # Debug print for input validation
-            print("Input validation:")
-            print(f"Chat histories: {len(chat_histories)} entries")
-            print(f"Topics: {all_topics}")
-
-            if not chat_histories or not all_topics:
-                print("Missing required input data")
-                return self._fallback_analytics()
-
-            # Debug the preprocessing step
-            try:
-                processed_histories = self._preprocess_chat_histories(chat_histories)
-                print("Successfully preprocessed chat histories")
-            except Exception as preprocess_error:
-                print(f"Error in preprocessing: {str(preprocess_error)}")
-                return self._fallback_analytics()
-
-            # Debug the prompt creation
-            try:
-                prompt = self._create_analytics_prompt(processed_histories, all_topics)
-                print("Successfully created prompt")
-                print("Prompt preview:", prompt[:200] + "...") # Print first 200 chars
-            except Exception as prompt_error:
-                print(f"Error in prompt creation: {str(prompt_error)}")
-                return self._fallback_analytics()
-
-            # Rest of the function remains the same
-            response = self.model.generate_content(
-                prompt,
-                generation_config=genai.GenerationConfig(
-                    response_mime_type="application/json",
-                    temperature=0.15
-                )
-            )
-            
-            if not response.text:
-                print("Empty response from Gemini")
-                return self._fallback_analytics()
-                
-            analytics = json.loads(response.text)
-            return analytics
-            
-        except Exception as e:
-            print(f"Error generating analytics: {str(e)}")
-            print(f"Error type: {type(e)}")
-            import traceback
-            print("Full traceback:", traceback.format_exc())
-            return self._fallback_analytics()
-
-    def _preprocess_chat_histories(self, chat_histories: List[Dict]) -> List[Dict]:
-        # """Preprocess chat histories to focus on relevant information."""
-        # processed = []
-        
-        # for chat in chat_histories:
-        #     print(str(chat["user_id"]))
-        #     processed_chat = {
-        #         "user_id": str(chat["user_id"]),
-        #         "messages": [
-        #             {
-        #                 "prompt": msg["prompt"],
-        #                 "response": msg["response"]
-        #             }
-        #             for msg in chat["messages"]
-        #         ]
-        #     }
-        #     processed.append(processed_chat)
-            
-        # return processed
-
-        # Code 2: 
-        """Preprocess chat histories to focus on relevant information."""
-        processed = []
-        
-        for chat in chat_histories:
-            # Convert ObjectId to string if it's an ObjectId
-            user_id = str(chat["user_id"]["$oid"]) if isinstance(chat["user_id"], dict) and "$oid" in chat["user_id"] else str(chat["user_id"])
-            
-            try:
-                processed_chat = {
-                    "user_id": user_id,
-                    "messages": [
-                        {
-                            "prompt": msg["prompt"],
-                            "response": msg["response"]
-                        }
-                        for msg in chat["messages"]
-                    ]
-                }
-                processed.append(processed_chat)
-                print(f"Successfully processed chat for user: {user_id}")
-            except Exception as e:
-                print(f"Error processing chat for user: {user_id}")
-                print(f"Error details: {str(e)}")
-                continue
-                
-        return processed
-
-    def _fallback_analytics(self) -> Dict:
-        # """Provide basic analytics in case of LLM processing failure."""
-        # return {
-        #     "topic_insights": [],
-        #     "student_insights": [],
-        #     "recommended_actions": [
-        #         {
-        #             "action": "Review analytics generation process",
-        #             "priority": "high",
-        #             "target_group": "system_administrators",
-        #             "reasoning": "Analytics generation failed",
-        #             "expected_impact": "Restore analytics functionality"
-        #         }
-        #     ]
-        # }
-        """Provide comprehensive fallback analytics that match our schema."""
-        return {
-            "topic_insights": [],
-            "student_insights": [],
-            "recommended_actions": [
-                {
-                    "action": "Review analytics generation process",
-                    "priority": "high",
-                    "target_group": "system_administrators",
-                    "reasoning": "Analytics generation failed",
-                    "expected_impact": "Restore analytics functionality"
-                }
-            ],
-            "course_health": {
-                "overall_engagement": 0,
-                "critical_topics": [],
-                "class_distribution": {
-                    "high_performers": 0,
-                    "average_performers": 0,
-                    "at_risk": 0
-                }
-            },
-            "intervention_metrics": {
-                "immediate_attention_needed": [],
-                "monitoring_required": []
-            }
-        }
-    
-# if __name__ == "__main__":
-#     # Example usage
-    
-    
-#     analytics_generator = NovaScholarAnalytics()
-#     analytics = analytics_generator.generate_analytics(chat_histories, all_topics)
+import json
+import typing_extensions as typing
+import google.generativeai as genai
+from typing import List, Dict, Any
+import numpy as np
+from collections import defaultdict
+
+from dotenv import load_dotenv
+import os
+import pymongo
+from pymongo import MongoClient
+
+load_dotenv()
+GEMINI_API_KEY = os.getenv('GEMINI_KEY')
+
+class EngagementMetrics(typing.TypedDict):
+    participation_level: str  # "high" | "medium" | "low"
+    question_quality: str     # "advanced" | "intermediate" | "basic"
+    concept_understanding: str  # "strong" | "moderate" | "needs_improvement"
+
+class StudentInsight(typing.TypedDict):
+    student_id: str
+    performance_level: str  # "high_performer" | "average" | "at_risk"
+    struggling_topics: list[str]
+    engagement_metrics: EngagementMetrics
+
+class TopicInsight(typing.TypedDict):
+    topic: str
+    difficulty_level: float  # 0 to 1
+    student_count: int
+    common_issues: list[str]
+    key_misconceptions: list[str]
+
+class RecommendedAction(typing.TypedDict):
+    action: str
+    priority: str  # "high" | "medium" | "low"
+    target_group: str  # "all_students" | "specific_students" | "faculty"
+    reasoning: str
+    expected_impact: str
+
+class ClassDistribution(typing.TypedDict):
+    high_performers: float
+    average_performers: float
+    at_risk: float
+
+class CourseHealth(typing.TypedDict):
+    overall_engagement: float  # 0 to 1
+    critical_topics: list[str]
+    class_distribution: ClassDistribution
+
+class InterventionMetrics(typing.TypedDict):
+    immediate_attention_needed: list[str]  # student_ids
+    monitoring_required: list[str]  # student_ids
+
+class AnalyticsResponse(typing.TypedDict):
+    topic_insights: list[TopicInsight]
+    student_insights: list[StudentInsight]
+    recommended_actions: list[RecommendedAction]
+    course_health: CourseHealth
+    intervention_metrics: InterventionMetrics
+
+
+
+class NovaScholarAnalytics:
+    def __init__(self, model_name: str = "gemini-1.5-flash"):
+        genai.configure(api_key=GEMINI_API_KEY)
+        self.model = genai.GenerativeModel(model_name)
+        
+    def _create_analytics_prompt(self, chat_histories: List[Dict], all_topics: List[str]) -> str:
+        """Creates a structured prompt for Gemini to analyze chat histories."""
+        # Prompt 1: 
+        # return f"""Analyze these student chat histories for a university course and provide detailed analytics.
+
+        # Context:
+        # - These are pre-class chat interactions between students and an AI tutor
+        # - Topics covered: {', '.join(all_topics)}
+
+        # Chat histories: {json.dumps(chat_histories, indent=2)}
+
+        # Return the analysis in JSON format matching this exact schema:
+        # {AnalyticsResponse.__annotations__}
+
+        # Ensure all numeric values are between 0 and 1 (accuracy upto 3 decimal places) where applicable.
+
+        # Important analysis guidelines:
+        # 1. Identify topics where students show confusion or ask multiple follow-up questions
+        # 2. Look for patterns in question types and complexity
+        # 3. Analyze response understanding based on follow-up questions
+        # 4. Consider both explicit and implicit signs of difficulty
+        # 5. Focus on concept relationships and prerequisite understanding"""
+
+        # Prompt 2: 
+        # return f"""Analyze the provided student chat histories for a university course and generate concise, actionable analytics.  
+
+        # Context:  
+        # - Chat histories: {json.dumps(chat_histories, indent=2)}
+        # - These are pre-class interactions between students and an AI tutor aimed at identifying learning difficulties and improving course delivery.  
+        # - Topics covered: {', '.join(all_topics)}.  
+
+        # Your task is to extract key insights that will help faculty address challenges effectively and enhance learning outcomes.  
+
+        # Output Format:  
+        # 1. Topics where students face significant difficulties:  
+        # - Provide a ranked list of topics where the majority of students are struggling, based on the frequency and nature of their questions or misconceptions.  
+        # - Include the percentage of students who found each topic challenging.  
+
+        # 2. AI-recommended actions for faculty:  
+        # - Suggest actionable steps to address the difficulties identified in each critical topic.  
+        # - Specify the priority of each action (high, medium, low) based on the urgency and impact.  
+        # - Explain the reasoning behind each recommendation and its expected impact on student outcomes.  
+
+        # 3. Student-specific analytics (focusing on at-risk students):  
+        # - Identify students categorized as "at-risk" based on their engagement levels, question complexity, and recurring struggles.  
+        # - For each at-risk student, list their top 3 struggling topics and their engagement metrics (participation level, concept understanding).  
+        # - Provide personalized recommendations for improving their understanding.  
+
+        # Guidelines for Analysis:  
+        # - Focus on actionable and concise insights rather than exhaustive details.  
+        # - Use both explicit (e.g., direct questions) and implicit (e.g., repeated follow-ups) cues to identify areas of difficulty.  
+        # - Prioritize topics with higher difficulty scores or more students struggling.  
+        # - Ensure numerical values (e.g., difficulty levels, percentages) are between 0 and 1 where applicable.  
+
+        # The response must be well-structured, concise, and highly actionable for faculty to implement improvements effectively."""  
+
+        # Prompt 3:
+        return f"""Analyze the provided student chat histories for a university course and generate concise, actionable analytics.
+        Context:
+        - Chat histories: {json.dumps(chat_histories, indent=2)}
+        - These are pre-class interactions between students and an AI tutor aimed at identifying learning difficulties and improving course delivery.
+        - Topics covered: {', '.join(all_topics)}.
+
+        Your task is to provide detailed analytics that will help faculty address challenges effectively and enhance learning outcomes.
+
+        Output Format (strictly follow this JSON structure):
+        {{
+        "topic_wise_insights": [
+            {{
+            "topic": "<string>",
+            "struggling_percentage": <number between 0 and 1>,
+            "key_issues": ["<string>", "<string>", ...],
+            "key_misconceptions": ["<string>", "<string>", ...],
+            "recommended_actions": {{
+                "description": "<string>",
+                "priority": "high|medium|low",
+                "expected_outcome": "<string>"
+            }}
+            }}
+        ],
+        "ai_recommended_actions": [
+        {{
+            "action": "<string>",
+            "priority": "high|medium|low",
+            "reasoning": "<string>",
+            "expected_outcome": "<string>",
+            "pedagogy_recommendations": {{
+                "methods": ["<string>", "<string>", ...],
+                "resources": ["<string>", "<string>", ...],
+                "expected_impact": "<string>"
+            }}
+        }}
+        ],
+        "student_analytics": [
+            {{
+            "student_id": "<string>",
+            "engagement_metrics": {{
+                "participation_level": <number between 0 and 1>,
+                "concept_understanding": "strong|moderate|needs_improvement",
+                "question_quality": "advanced|intermediate|basic"
+            }},
+            "struggling_topics": ["<string>", "<string>", ...],
+            "personalized_recommendation": "<string>"
+            }}
+        ]
+        }}
+
+        Guidelines for Analysis:
+        - Focus on actionable and concise insights rather than exhaustive details.
+        - Use both explicit (e.g., direct questions) and implicit (e.g., repeated follow-ups) cues to identify areas of difficulty.
+        - Prioritize topics with higher difficulty scores or more students struggling.
+        - Ensure numerical values (e.g., difficulty levels, percentages) are between 0 and 1 where applicable.
+        - Make sure to include All** students in the analysis, not just a subset.
+        - for the ai_recommended_actions:
+            - Prioritize pedagogy recommendations for critical topics with the high difficulty scores or struggling percentages.
+            - For each action:
+                - Include specific teaching methods (e.g., interactive discussions or quizzes, problem-based learning, practical examples etc).
+                - Recommend supporting resources (e.g., videos, handouts, simulations).
+                - Provide reasoning for the recommendation and the expected outcomes for student learning.
+                - Example:
+                - **Action:** Conduct an interactive problem-solving session on "<Topic Name>".
+                - **Reasoning:** Students showed difficulty in applying concepts to practical problems.
+                - **Expected Outcome:** Improved practical understanding and application of the topic.
+                - **Pedagogy Recommendations:**
+                    - **Methods:** Group discussions, real-world case studies.
+                    - **Resources:** Online interactive tools, relevant case studies, video walkthroughs.
+                    - **Expected Impact:** Enhance conceptual clarity by 40% and practical application by 30%. 
+
+        The response must adhere strictly to the above JSON structure, with all fields populated appropriately."""
+
+    
+    def _calculate_class_distribution(self, analytics: Dict) -> Dict:
+        """Calculate the distribution of students across performance levels."""
+        try:
+            total_students = len(analytics.get("student_insights", []))
+            if total_students == 0:
+                return {
+                    "high_performers": 0,
+                    "average_performers": 0,
+                    "at_risk": 0
+                }
+            
+            distribution = defaultdict(int)
+            
+            for student in analytics.get("student_insights", []):
+                performance_level = student.get("performance_level", "average")
+                # Map performance levels to our three categories
+                if performance_level in ["excellent", "high", "high_performer"]:
+                    distribution["high_performers"] += 1
+                elif performance_level in ["struggling", "low", "at_risk"]:
+                    distribution["at_risk"] += 1
+                else:
+                    distribution["average_performers"] += 1
+            
+            # Convert to percentages
+            return {
+                level: count/total_students 
+                for level, count in distribution.items()
+            }
+        except Exception as e:
+            print(f"Error calculating class distribution: {str(e)}")
+            return {
+                "high_performers": 0,
+                "average_performers": 0,
+                "at_risk": 0
+            }
+
+    def _identify_urgent_cases(self, analytics: Dict) -> List[str]:
+        """Identify students needing immediate attention."""
+        try:
+            urgent_cases = []
+            for student in analytics.get("student_insights", []):
+                student_id = student.get("student_id")
+                if not student_id:
+                    continue
+                    
+                # Check multiple risk factors
+                risk_factors = 0
+                
+                # Factor 1: Performance level
+                if student.get("performance_level") in ["struggling", "at_risk", "low"]:
+                    risk_factors += 1
+                    
+                # Factor 2: Number of struggling topics
+                if len(student.get("struggling_topics", [])) >= 2:
+                    risk_factors += 1
+                    
+                # Factor 3: Engagement metrics
+                engagement = student.get("engagement_metrics", {})
+                if (engagement.get("participation_level") == "low" or 
+                    engagement.get("concept_understanding") == "needs_improvement"):
+                    risk_factors += 1
+                
+                # If student has multiple risk factors, add to urgent cases
+                if risk_factors >= 2:
+                    urgent_cases.append(student_id)
+            
+            return urgent_cases
+        except Exception as e:
+            print(f"Error identifying urgent cases: {str(e)}")
+            return []
+
+    def _identify_monitoring_cases(self, analytics: Dict) -> List[str]:
+        """Identify students who need monitoring but aren't urgent cases."""
+        try:
+            monitoring_cases = []
+            urgent_cases = set(self._identify_urgent_cases(analytics))
+            
+            for student in analytics.get("student_insights", []):
+                student_id = student.get("student_id")
+                if not student_id or student_id in urgent_cases:
+                    continue
+                
+                # Check monitoring criteria
+                monitoring_needed = False
+                
+                # Criterion 1: Has some struggling topics but not enough for urgent
+                if len(student.get("struggling_topics", [])) == 1:
+                    monitoring_needed = True
+                    
+                # Criterion 2: Medium-low engagement
+                engagement = student.get("engagement_metrics", {})
+                if engagement.get("participation_level") == "medium":
+                    monitoring_needed = True
+                    
+                # Criterion 3: Recent performance decline
+                if student.get("performance_level") == "average":
+                    monitoring_needed = True
+                
+                if monitoring_needed:
+                    monitoring_cases.append(student_id)
+            
+            return monitoring_cases
+        except Exception as e:
+            print(f"Error identifying monitoring cases: {str(e)}")
+            return []
+
+    def _identify_critical_topics(self, analytics: Dict) -> List[str]:
+        """
+        Identify critical topics that need attention based on multiple factors.
+        Returns a list of topic names that are considered critical.
+        """
+        try:
+            critical_topics = []
+            topics = analytics.get("topic_insights", [])
+            
+            for topic in topics:
+                if not isinstance(topic, dict):
+                    continue
+                    
+                # Initialize score for topic criticality
+                critical_score = 0
+                
+                # Factor 1: High difficulty level
+                difficulty_level = topic.get("difficulty_level", 0)
+                if difficulty_level > 0.7:
+                    critical_score += 2
+                elif difficulty_level > 0.5:
+                    critical_score += 1
+                    
+                # Factor 2: Number of students struggling
+                student_count = topic.get("student_count", 0)
+                total_students = len(analytics.get("student_insights", []))
+                if total_students > 0:
+                    struggle_ratio = student_count / total_students
+                    if struggle_ratio > 0.5:
+                        critical_score += 2
+                    elif struggle_ratio > 0.3:
+                        critical_score += 1
+                        
+                # Factor 3: Number of common issues
+                if len(topic.get("common_issues", [])) > 2:
+                    critical_score += 1
+                    
+                # Factor 4: Number of key misconceptions
+                if len(topic.get("key_misconceptions", [])) > 1:
+                    critical_score += 1
+                
+                # If topic exceeds threshold, mark as critical
+                if critical_score >= 3:
+                    critical_topics.append(topic.get("topic", "Unknown Topic"))
+            
+            return critical_topics
+            
+        except Exception as e:
+            print(f"Error identifying critical topics: {str(e)}")
+            return []
+
+    def _calculate_engagement(self, analytics: Dict) -> Dict:
+        """
+        Calculate detailed engagement metrics across all students.
+        Returns a dictionary with engagement statistics.
+        """
+        try:
+            total_students = len(analytics.get("student_insights", []))
+            if total_students == 0:
+                return {
+                    "total_students": 0,
+                    "overall_score": 0,
+                    "engagement_distribution": {
+                        "high": 0,
+                        "medium": 0,
+                        "low": 0
+                    },
+                    "participation_metrics": {
+                        "average_topics_per_student": 0,
+                        "active_participants": 0
+                    }
+                }
+            
+            engagement_levels = defaultdict(int)
+            total_topics_engaged = 0
+            active_participants = 0
+            
+            for student in analytics.get("student_insights", []):
+                # Get engagement metrics
+                metrics = student.get("engagement_metrics", {})
+                
+                # Calculate participation level
+                participation = metrics.get("participation_level", "low").lower()
+                engagement_levels[participation] += 1
+                
+                # Count topics student is engaged with
+                topics_count = len(student.get("struggling_topics", []))
+                total_topics_engaged += topics_count
+                
+                # Count active participants (students engaging with any topics)
+                if topics_count > 0:
+                    active_participants += 1
+            
+            # Calculate overall engagement score (0-1)
+            weighted_score = (
+                (engagement_levels["high"] * 1.0 + 
+                engagement_levels["medium"] * 0.6 + 
+                engagement_levels["low"] * 0.2) / total_students
+            )
+            
+            return {
+                "total_students": total_students,
+                "overall_score": round(weighted_score, 2),
+                "engagement_distribution": {
+                    level: count/total_students 
+                    for level, count in engagement_levels.items()
+                },
+                "participation_metrics": {
+                    "average_topics_per_student": round(total_topics_engaged / total_students, 2),
+                    "active_participants_ratio": round(active_participants / total_students, 2)
+                }
+            }
+            
+        except Exception as e:
+            print(f"Error calculating engagement: {str(e)}")
+            return {
+                "total_students": 0,
+                "overall_score": 0,
+                "engagement_distribution": {
+                    "high": 0,
+                    "medium": 0,
+                    "low": 0
+                },
+                "participation_metrics": {
+                    "average_topics_per_student": 0,
+                    "active_participants_ratio": 0
+                }
+            }
+
+    def _process_gemini_response(self, response: str) -> Dict:
+        """Process and validate Gemini's response."""
+        # try:
+        #     analytics = json.loads(response)
+        #     return self._enrich_analytics(analytics)
+        # except json.JSONDecodeError as e:
+        #     print(f"Error decoding Gemini response: {e}")
+        #     return self._fallback_analytics()
+        try:
+            # Parse JSON response
+            analytics = json.loads(response)
+            
+            # Validate required fields exist
+            required_fields = {
+                "topic_insights": [],
+                "student_insights": [],
+                "recommended_actions": []
+            }
+            
+            # Ensure all required fields exist with default values
+            for field, default_value in required_fields.items():
+                if field not in analytics or not analytics[field]:
+                    analytics[field] = default_value
+            
+            # Now enrich the validated analytics
+            return self._enrich_analytics(analytics)
+            
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            print(f"Error processing Gemini response: {str(e)}")
+            print(f"Raw response: {response}")
+            return self._fallback_analytics()
+
+    def _enrich_analytics(self, analytics: Dict) -> Dict:
+        """Add derived insights and metrics to the analytics."""
+        # Add overall course health metrics
+        analytics["course_health"] = {
+            "overall_engagement": self._calculate_engagement(analytics),
+            "critical_topics": self._identify_critical_topics(analytics),
+            "class_distribution": self._calculate_class_distribution(analytics)
+        }
+        
+        # Add intervention urgency scores
+        analytics["intervention_metrics"] = {
+            "immediate_attention_needed": self._identify_urgent_cases(analytics),
+            "monitoring_required": self._identify_monitoring_cases(analytics)
+        }
+        
+        return analytics
+
+    def _calculate_engagement(self, analytics: Dict) -> Dict:
+        # """Calculate overall engagement metrics."""
+        # total_students = len(analytics["student_insights"])
+        # engagement_levels = defaultdict(int)
+        
+        # for student in analytics["student_insights"]:
+        #     engagement_levels[student["engagement_metrics"]["participation_level"]] += 1
+            
+        # return {
+        #     "total_students": total_students,
+        #     "engagement_distribution": {
+        #         level: count/total_students 
+        #         for level, count in engagement_levels.items()
+        #     }
+        # }
+        """Calculate overall engagement metrics with defensive programming."""
+        try:
+            total_students = len(analytics.get("student_insights", []))
+            if total_students == 0:
+                return {
+                    "total_students": 0,
+                    "engagement_distribution": {
+                        "high": 0,
+                        "medium": 0,
+                        "low": 0
+                    }
+                }
+            
+            engagement_levels = defaultdict(int)
+            
+            for student in analytics.get("student_insights", []):
+                metrics = student.get("engagement_metrics", {})
+                level = metrics.get("participation_level", "low")
+                engagement_levels[level] += 1
+                
+            return {
+                "total_students": total_students,
+                "engagement_distribution": {
+                    level: count/total_students 
+                    for level, count in engagement_levels.items()
+                }
+            }
+        except Exception as e:
+            print(f"Error calculating engagement: {str(e)}")
+            return {
+                "total_students": 0,
+                "engagement_distribution": {
+                    "high": 0,
+                    "medium": 0,
+                    "low": 0
+                }
+            }
+
+    def _identify_critical_topics(self, analytics: Dict) -> List[Dict]:
+        # """Identify topics needing immediate attention."""
+        # return [
+        #     topic for topic in analytics["topic_insights"]
+        #     if topic["difficulty_level"] > 0.7 or 
+        #     len(topic["common_issues"]) > 2
+        # ]
+        """Identify topics needing immediate attention with defensive programming."""
+        try:
+            return [
+                topic for topic in analytics.get("topic_insights", [])
+                if topic.get("difficulty_level", 0) > 0.7 or 
+                len(topic.get("common_issues", [])) > 2
+            ]
+        except Exception as e:
+            print(f"Error identifying critical topics: {str(e)}")
+            return []
+
+    def generate_analytics(self, chat_histories: List[Dict], all_topics: List[str]) -> Dict:
+        # Method 1: (caused key 'student_insights' error): 
+        # """Main method to generate analytics from chat histories."""
+        # # Preprocess chat histories
+        # processed_histories = self._preprocess_chat_histories(chat_histories)
+        
+        # # Create and send prompt to Gemini
+        # prompt = self._create_analytics_prompt(processed_histories, all_topics)
+        # response = self.model.generate_content(
+        #     prompt, 
+        #     generation_config=genai.GenerationConfig(
+        #         response_mime_type="application/json",
+        #         response_schema=AnalyticsResponse
+        #     )
+        # )
+        
+        # # # Process and enrich analytics
+        # # analytics = self._process_gemini_response(response.text)
+        # # return analytics
+        # # Process, validate, and enrich the response
+        # analytics = self._process_gemini_response(response.text)
+    
+        # # Then cast it to satisfy the type checker
+        # return typing.cast(AnalyticsResponse, analytics)
+
+        # Method 2 (possible fix): 
+        # """Main method to generate analytics with better error handling."""
+        # try:
+        #     processed_histories = self._preprocess_chat_histories(chat_histories)
+        #     prompt = self._create_analytics_prompt(processed_histories, all_topics)
+            
+        #     response = self.model.generate_content(
+        #         prompt,
+        #         generation_config=genai.GenerationConfig(
+        #             response_mime_type="application/json",
+        #             temperature=0.15
+        #             # response_schema=AnalyticsResponse
+        #         )
+        #     )
+            
+        #     if not response.text:
+        #         print("Empty response from Gemini")
+        #         return self._fallback_analytics()
+                
+        #     # analytics = self._process_gemini_response(response.text)
+        #     # return typing.cast(AnalyticsResponse, analytics)
+        #     # return response.text;
+        #     analytics = json.loads(response.text)
+        #     return analytics
+        
+        # except Exception as e:
+        #     print(f"Error generating analytics: {str(e)}")
+        #     return self._fallback_analytics()
+
+
+        # Debugging code: 
+        """Main method to generate analytics with better error handling."""
+        try:
+            # Debug print for input validation
+            print("Input validation:")
+            print(f"Chat histories: {len(chat_histories)} entries")
+            print(f"Topics: {all_topics}")
+
+            if not chat_histories or not all_topics:
+                print("Missing required input data")
+                return self._fallback_analytics()
+
+            # Debug the preprocessing step
+            try:
+                processed_histories = self._preprocess_chat_histories(chat_histories)
+                print("Successfully preprocessed chat histories")
+            except Exception as preprocess_error:
+                print(f"Error in preprocessing: {str(preprocess_error)}")
+                return self._fallback_analytics()
+
+            # Debug the prompt creation
+            try:
+                prompt = self._create_analytics_prompt(processed_histories, all_topics)
+                print("Successfully created prompt")
+                print("Prompt preview:", prompt[:200] + "...") # Print first 200 chars
+            except Exception as prompt_error:
+                print(f"Error in prompt creation: {str(prompt_error)}")
+                return self._fallback_analytics()
+
+            # Rest of the function remains the same
+            response = self.model.generate_content(
+                prompt,
+                generation_config=genai.GenerationConfig(
+                    response_mime_type="application/json",
+                    temperature=0.15
+                )
+            )
+            
+            if not response.text:
+                print("Empty response from Gemini")
+                return self._fallback_analytics()
+                
+            analytics = json.loads(response.text)
+            return analytics
+            
+        except Exception as e:
+            print(f"Error generating analytics: {str(e)}")
+            print(f"Error type: {type(e)}")
+            import traceback
+            print("Full traceback:", traceback.format_exc())
+            return self._fallback_analytics()
+
+    def _preprocess_chat_histories(self, chat_histories: List[Dict]) -> List[Dict]:
+        # """Preprocess chat histories to focus on relevant information."""
+        # processed = []
+        
+        # for chat in chat_histories:
+        #     print(str(chat["user_id"]))
+        #     processed_chat = {
+        #         "user_id": str(chat["user_id"]),
+        #         "messages": [
+        #             {
+        #                 "prompt": msg["prompt"],
+        #                 "response": msg["response"]
+        #             }
+        #             for msg in chat["messages"]
+        #         ]
+        #     }
+        #     processed.append(processed_chat)
+            
+        # return processed
+
+        # Code 2: 
+        """Preprocess chat histories to focus on relevant information."""
+        processed = []
+        
+        for chat in chat_histories:
+            # Convert ObjectId to string if it's an ObjectId
+            user_id = str(chat["user_id"]["$oid"]) if isinstance(chat["user_id"], dict) and "$oid" in chat["user_id"] else str(chat["user_id"])
+            
+            try:
+                processed_chat = {
+                    "user_id": user_id,
+                    "messages": [
+                        {
+                            "prompt": msg["prompt"],
+                            "response": msg["response"]
+                        }
+                        for msg in chat["messages"]
+                    ]
+                }
+                processed.append(processed_chat)
+                print(f"Successfully processed chat for user: {user_id}")
+            except Exception as e:
+                print(f"Error processing chat for user: {user_id}")
+                print(f"Error details: {str(e)}")
+                continue
+                
+        return processed
+
+    def _fallback_analytics(self) -> Dict:
+        # """Provide basic analytics in case of LLM processing failure."""
+        # return {
+        #     "topic_insights": [],
+        #     "student_insights": [],
+        #     "recommended_actions": [
+        #         {
+        #             "action": "Review analytics generation process",
+        #             "priority": "high",
+        #             "target_group": "system_administrators",
+        #             "reasoning": "Analytics generation failed",
+        #             "expected_impact": "Restore analytics functionality"
+        #         }
+        #     ]
+        # }
+        """Provide comprehensive fallback analytics that match our schema."""
+        return {
+            "topic_insights": [],
+            "student_insights": [],
+            "recommended_actions": [
+                {
+                    "action": "Review analytics generation process",
+                    "priority": "high",
+                    "target_group": "system_administrators",
+                    "reasoning": "Analytics generation failed",
+                    "expected_impact": "Restore analytics functionality"
+                }
+            ],
+            "course_health": {
+                "overall_engagement": 0,
+                "critical_topics": [],
+                "class_distribution": {
+                    "high_performers": 0,
+                    "average_performers": 0,
+                    "at_risk": 0
+                }
+            },
+            "intervention_metrics": {
+                "immediate_attention_needed": [],
+                "monitoring_required": []
+            }
+        }
+    
+# if __name__ == "__main__":
+#     # Example usage
+    
+    
+#     analytics_generator = NovaScholarAnalytics()
+#     analytics = analytics_generator.generate_analytics(chat_histories, all_topics)
 #     print(json.dumps(analytics, indent=2))

pre_class_analytics4.py

@@ -1,592 +1,592 @@
-import pandas as pd
-import numpy as np
-from datetime import datetime
-from typing import List, Dict, Any, Tuple
-import spacy
-from collections import Counter, defaultdict
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-from textblob import TextBlob
-import networkx as nx
-from scipy import stats
-import logging
-import json
-from dataclasses import dataclass
-from enum import Enum
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-class TopicDifficulty(Enum):
-    EASY = "easy"
-    MODERATE = "moderate"
-    DIFFICULT = "difficult"
-    VERY_DIFFICULT = "very_difficult"
-
-
-@dataclass
-class QuestionMetrics:
-    complexity_score: float
-    follow_up_count: int
-    clarification_count: int
-    time_spent: float
-    sentiment_score: float
-
-@dataclass
-class TopicInsights:
-    difficulty_level: TopicDifficulty
-    common_confusion_points: List[str]
-    question_patterns: List[str]
-    time_distribution: Dict[str, float]
-    engagement_metrics: Dict[str, float]
-    recommended_focus_areas: List[str]
-
-    def to_dict(self):
-        return {
-            "difficulty_level": self.difficulty_level.value,  # Convert enum to its value
-            "common_confusion_points": self.common_confusion_points,
-            "question_patterns": self.question_patterns,
-            "time_distribution": {str(k): v for k, v in self.time_distribution.items()},
-            "engagement_metrics": self.engagement_metrics,
-            "recommended_focus_areas": self.recommended_focus_areas,
-        }
-
-class PreClassAnalytics:
-    def __init__(self, nlp_model: str = "en_core_web_lg"):
-        """Initialize the analytics system with necessary components."""
-        self.nlp = spacy.load(nlp_model)
-        self.question_indicators = {
-            "what", "why", "how", "when", "where", "which", "who", 
-            "whose", "whom", "can", "could", "would", "will", "explain"
-        }
-        self.confusion_indicators = {
-            "confused", "don't understand", "unclear", "not clear",
-            "stuck", "difficult", "hard", "help", "explain again"
-        }
-        self.follow_up_indicators = {
-            "also", "another", "additionally", "furthermore", "moreover",
-            "besides", "related", "similarly", "again"
-        }
-        
-    def preprocess_chat_history(self, chat_history: List[Dict]) -> pd.DataFrame:
-        """Convert chat history to DataFrame with enhanced features."""
-        messages = []
-        for chat in chat_history:
-            user_id = chat['user_id']['$oid']
-            for msg in chat['messages']:
-                try:
-                    # Ensure the timestamp is in the correct format
-                    if isinstance(msg['timestamp'], dict) and '$date' in msg['timestamp']:
-                        timestamp = pd.to_datetime(msg['timestamp']['$date'])
-                    elif isinstance(msg['timestamp'], str):
-                        timestamp = pd.to_datetime(msg['timestamp'])
-                    else:
-                        raise ValueError("Invalid timestamp format")
-                except Exception as e:
-                    print(f"Error parsing timestamp: {msg['timestamp']}, error: {e}")
-                    timestamp = pd.NaT  # Use NaT (Not a Time) for invalid timestamps
-                
-                messages.append({
-                    'user_id': user_id,
-                    'timestamp': timestamp,
-                    'prompt': msg['prompt'],
-                    'response': msg['response'],
-                    'is_question': any(q in msg['prompt'].lower() for q in self.question_indicators),
-                    'shows_confusion': any(c in msg['prompt'].lower() for c in self.confusion_indicators),
-                    'is_followup': any(f in msg['prompt'].lower() for f in self.follow_up_indicators)
-                })
-        
-        df = pd.DataFrame(messages)
-        df['sentiment'] = df['prompt'].apply(lambda x: TextBlob(x).sentiment.polarity)
-        return df
-
-    def extract_topic_hierarchies(self, df: pd.DataFrame) -> Dict[str, List[str]]:
-        """Extract hierarchical topic relationships from conversations."""
-        topic_hierarchy = defaultdict(list)
-        
-        for _, row in df.iterrows():
-            doc = self.nlp(row['prompt'])
-            
-            # Extract main topics and subtopics using noun chunks and dependencies
-            main_topics = []
-            subtopics = []
-            
-            for chunk in doc.noun_chunks:
-                if chunk.root.dep_ in ('nsubj', 'dobj'):
-                    main_topics.append(chunk.text.lower())
-                else:
-                    subtopics.append(chunk.text.lower())
-            
-            # Build hierarchy
-            for main_topic in main_topics:
-                topic_hierarchy[main_topic].extend(subtopics)
-        
-        # Clean and deduplicate
-        return {k: list(set(v)) for k, v in topic_hierarchy.items()}
-
-    def analyze_topic_difficulty(self, df: pd.DataFrame, topic: str) -> TopicDifficulty:
-        """Determine topic difficulty based on various metrics."""
-        topic_msgs = df[df['prompt'].str.contains(topic, case=False)]
-        
-        # Calculate difficulty indicators
-        confusion_rate = topic_msgs['shows_confusion'].mean()
-        question_rate = topic_msgs['is_question'].mean()
-        follow_up_rate = topic_msgs['is_followup'].mean()
-        avg_sentiment = topic_msgs['sentiment'].mean()
-        
-        # Calculate composite difficulty score
-        difficulty_score = (
-            confusion_rate * 0.4 +
-            question_rate * 0.3 +
-            follow_up_rate * 0.2 +
-            (1 - (avg_sentiment + 1) / 2) * 0.1
-        )
-        
-        # Map score to difficulty level
-        if difficulty_score < 0.3:
-            return TopicDifficulty.EASY
-        elif difficulty_score < 0.5:
-            return TopicDifficulty.MODERATE
-        elif difficulty_score < 0.7:
-            return TopicDifficulty.DIFFICULT
-        else:
-            return TopicDifficulty.VERY_DIFFICULT
-
-    def identify_confusion_patterns(self, df: pd.DataFrame, topic: str) -> List[str]:
-        """Identify common patterns in student confusion."""
-        confused_msgs = df[
-            (df['prompt'].str.contains(topic, case=False)) & 
-            (df['shows_confusion'])
-        ]['prompt']
-        
-        patterns = []
-        for msg in confused_msgs:
-            doc = self.nlp(msg)
-            
-            # Extract key phrases around confusion indicators
-            for sent in doc.sents:
-                for token in sent:
-                    if token.text.lower() in self.confusion_indicators:
-                        # Get context window around confusion indicator
-                        context = sent.text
-                        patterns.append(context)
-        
-        # Group similar patterns
-        if patterns:
-            vectorizer = TfidfVectorizer(ngram_range=(1, 3))
-            tfidf_matrix = vectorizer.fit_transform(patterns)
-            similarity_matrix = cosine_similarity(tfidf_matrix)
-            
-            # Cluster similar patterns
-            G = nx.Graph()
-            for i in range(len(patterns)):
-                for j in range(i + 1, len(patterns)):
-                    if similarity_matrix[i][j] > 0.5:  # Similarity threshold
-                        G.add_edge(i, j)
-            
-            # Extract representative patterns from each cluster
-            clusters = list(nx.connected_components(G))
-            return [patterns[min(cluster)] for cluster in clusters]
-        
-        return []
-
-    def analyze_question_patterns(self, df: pd.DataFrame, topic: str) -> List[str]:
-        """Analyze patterns in student questions about the topic."""
-        topic_questions = df[
-            (df['prompt'].str.contains(topic, case=False)) & 
-            (df['is_question'])
-        ]['prompt']
-        
-        question_types = defaultdict(list)
-        for question in topic_questions:
-            doc = self.nlp(question)
-            
-            # Categorize questions
-            if any(token.text.lower() in {"what", "define", "explain"} for token in doc):
-                question_types["conceptual"].append(question)
-            elif any(token.text.lower() in {"how", "steps", "process"} for token in doc):
-                question_types["procedural"].append(question)
-            elif any(token.text.lower() in {"why", "reason", "because"} for token in doc):
-                question_types["reasoning"].append(question)
-            else:
-                question_types["other"].append(question)
-        
-        # Extract patterns from each category
-        patterns = []
-        for category, questions in question_types.items():
-            if questions:
-                vectorizer = TfidfVectorizer(ngram_range=(1, 3))
-                tfidf_matrix = vectorizer.fit_transform(questions)
-                
-                # Get most representative questions
-                feature_array = np.mean(tfidf_matrix.toarray(), axis=0)
-                tfidf_sorting = np.argsort(feature_array)[::-1]
-                features = vectorizer.get_feature_names_out()
-                
-                patterns.append(f"{category}: {' '.join(features[tfidf_sorting[:3]])}")
-        
-        return patterns
-
-    def analyze_time_distribution(self, df: pd.DataFrame, topic: str) -> Dict[str, float]:
-        """Analyze time spent on different aspects of the topic."""
-        topic_msgs = df[df['prompt'].str.contains(topic, case=False)].copy()
-        if len(topic_msgs) < 2:
-            return {}
-        
-        topic_msgs['time_diff'] = topic_msgs['timestamp'].diff()
-        
-        # Calculate time distribution
-        distribution = {
-            'total_time': topic_msgs['time_diff'].sum().total_seconds() / 60,
-            'avg_time_per_message': topic_msgs['time_diff'].mean().total_seconds() / 60,
-            'max_time_gap': topic_msgs['time_diff'].max().total_seconds() / 60,
-            'time_spent_on_questions': topic_msgs[topic_msgs['is_question']]['time_diff'].sum().total_seconds() / 60,
-            'time_spent_on_confusion': topic_msgs[topic_msgs['shows_confusion']]['time_diff'].sum().total_seconds() / 60
-        }
-        
-        return distribution
-
-    def calculate_engagement_metrics(self, df: pd.DataFrame, topic: str) -> Dict[str, float]:
-        """Calculate student engagement metrics for the topic."""
-        topic_msgs = df[df['prompt'].str.contains(topic, case=False)]
-        
-        metrics = {
-            'message_count': len(topic_msgs),
-            'question_ratio': topic_msgs['is_question'].mean(),
-            'confusion_ratio': topic_msgs['shows_confusion'].mean(),
-            'follow_up_ratio': topic_msgs['is_followup'].mean(),
-            'avg_sentiment': topic_msgs['sentiment'].mean(),
-            'engagement_score': 0.0  # Will be calculated below
-        }
-        
-        # Calculate engagement score
-        metrics['engagement_score'] = (
-            metrics['message_count'] * 0.3 +
-            metrics['question_ratio'] * 0.25 +
-            metrics['follow_up_ratio'] * 0.25 +
-            (metrics['avg_sentiment'] + 1) / 2 * 0.2  # Normalize sentiment to 0-1
-        )
-        
-        return metrics
-
-    def generate_topic_insights(self, df: pd.DataFrame, topic: str) -> TopicInsights:
-        """Generate comprehensive insights for a topic."""
-        difficulty = self.analyze_topic_difficulty(df, topic)
-        confusion_points = self.identify_confusion_patterns(df, topic)
-        question_patterns = self.analyze_question_patterns(df, topic)
-        time_distribution = self.analyze_time_distribution(df, topic)
-        engagement_metrics = self.calculate_engagement_metrics(df, topic)
-        
-        # Generate recommended focus areas based on insights
-        focus_areas = []
-        
-        if difficulty in (TopicDifficulty.DIFFICULT, TopicDifficulty.VERY_DIFFICULT):
-            focus_areas.append("Fundamental concept reinforcement needed")
-        
-        if confusion_points:
-            focus_areas.append(f"Address common confusion around: {', '.join(confusion_points[:3])}")
-        
-        if engagement_metrics['confusion_ratio'] > 0.3:
-            focus_areas.append("Consider alternative teaching approaches")
-        
-        if time_distribution.get('time_spent_on_questions', 0) > time_distribution.get('total_time', 0) * 0.5:
-            focus_areas.append("More practical examples or demonstrations needed")
-        
-        return TopicInsights(
-            difficulty_level=difficulty,
-            common_confusion_points=confusion_points,
-            question_patterns=question_patterns,
-            time_distribution=time_distribution,
-            engagement_metrics=engagement_metrics,
-            recommended_focus_areas=focus_areas
-        )
-
-    def analyze_student_progress(self, df: pd.DataFrame) -> Dict[str, Any]:
-        """Analyze individual student progress and learning patterns."""
-        student_progress = {}
-        
-        for student_id in df['user_id'].unique():
-            student_msgs = df[df['user_id'] == student_id]
-            
-            # Calculate student-specific metrics
-            progress = {
-                'total_messages': len(student_msgs),
-                'questions_asked': student_msgs['is_question'].sum(),
-                'confusion_instances': student_msgs['shows_confusion'].sum(),
-                'avg_sentiment': student_msgs['sentiment'].mean(),
-                'topic_engagement': {},
-                'learning_pattern': self._identify_learning_pattern(student_msgs)
-            }
-            
-            # Analyze topic-specific engagement
-            topics = self.extract_topic_hierarchies(student_msgs)
-            for topic in topics:
-                topic_msgs = student_msgs[student_msgs['prompt'].str.contains(topic, case=False)]
-                progress['topic_engagement'][topic] = {
-                    'message_count': len(topic_msgs),
-                    'confusion_rate': topic_msgs['shows_confusion'].mean(),
-                    'sentiment_trend': stats.linregress(
-                        range(len(topic_msgs)),
-                        topic_msgs['sentiment']
-                    ).slope
-                }
-            
-            student_progress[student_id] = progress
-        
-        return student_progress
-
-    def _identify_learning_pattern(self, student_msgs: pd.DataFrame) -> str:
-        """Identify student's learning pattern based on their interaction style."""
-        # Calculate key metrics
-        question_ratio = student_msgs['is_question'].mean()
-        confusion_ratio = student_msgs['shows_confusion'].mean()
-        follow_up_ratio = student_msgs['is_followup'].mean()
-        sentiment_trend = stats.linregress(
-            range(len(student_msgs)),
-            student_msgs['sentiment']
-        ).slope
-        
-        # Identify pattern
-        if question_ratio > 0.6:
-            return "Inquisitive Learner"
-        elif confusion_ratio > 0.4:
-            return "Needs Additional Support"
-        elif follow_up_ratio > 0.5:
-            return "Deep Dive Learner"
-        elif sentiment_trend > 0:
-            return "Progressive Learner"
-        else:
-            return "Steady Learner"
-
-    def generate_comprehensive_report(self, chat_history: List[Dict]) -> Dict[str, Any]:
-        """Generate a comprehensive analytics report."""
-        # Preprocess chat history
-        df = self.preprocess_chat_history(chat_history)
-        
-        # Extract topics
-        topics = self.extract_topic_hierarchies(df)
-        
-        report = {
-            'topics': {},
-            'student_progress': self.analyze_student_progress(df),
-            'overall_metrics': {
-                'total_conversations': len(df),
-                'unique_students': df['user_id'].nunique(),
-                'avg_sentiment': df['sentiment'].mean(),
-                'most_discussed_topics': Counter(
-                    topic for topics_list in topics.values() 
-                    for topic in topics_list
-                ).most_common(5)
-            }
-        }
-        
-        # Generate topic-specific insights
-        for main_topic, subtopics in topics.items():
-            subtopic_insights = {}
-            for subtopic in subtopics:
-                subtopic_insights[subtopic] = {
-                    'insights': self.generate_topic_insights(df, subtopic),
-                    'related_topics': [t for t in subtopics if t != subtopic],
-                    'student_engagement': {
-                        student_id: self.calculate_engagement_metrics(
-                            df[df['user_id'] == student_id], 
-                            subtopic
-                        )
-                        for student_id in df['user_id'].unique()
-                    }
-                }
-            
-            report['topics'][main_topic] = {
-                'insights': self.generate_topic_insights(df, main_topic),
-                'subtopics': subtopic_insights,
-                'topic_relationships': {
-                    'hierarchy_depth': len(subtopics),
-                    'connection_strength': self._calculate_topic_connections(df, main_topic, subtopics),
-                    'progression_path': self._identify_topic_progression(df, main_topic, subtopics)
-                }
-            }
-        
-        # Add temporal analysis
-        report['temporal_analysis'] = {
-            'daily_engagement': df.groupby(df['timestamp'].dt.date).agg({
-                'user_id': 'count',
-                'is_question': 'sum',
-                'shows_confusion': 'sum',
-                'sentiment': 'mean'
-            }).to_dict(),
-            'peak_activity_hours': df.groupby(df['timestamp'].dt.hour)['user_id'].count().nlargest(3).to_dict(),
-            'learning_trends': self._analyze_learning_trends(df)
-        }
-        
-        # Add recommendations
-        report['recommendations'] = self._generate_recommendations(report)
-        
-        return report
-
-    def _calculate_topic_connections(self, df: pd.DataFrame, main_topic: str, subtopics: List[str]) -> Dict[str, float]:
-        """Calculate connection strength between topics based on co-occurrence."""
-        connections = {}
-        main_topic_msgs = df[df['prompt'].str.contains(main_topic, case=False)]
-        
-        for subtopic in subtopics:
-            cooccurrence = df[
-                df['prompt'].str.contains(main_topic, case=False) & 
-                df['prompt'].str.contains(subtopic, case=False)
-            ].shape[0]
-            
-            connection_strength = cooccurrence / len(main_topic_msgs) if len(main_topic_msgs) > 0 else 0
-            connections[subtopic] = connection_strength
-        
-        return connections
-
-    def _identify_topic_progression(self, df: pd.DataFrame, main_topic: str, subtopics: List[str]) -> List[str]:
-        """Identify optimal topic progression path based on student interactions."""
-        topic_difficulties = {}
-        
-        for subtopic in subtopics:
-            difficulty = self.analyze_topic_difficulty(df, subtopic)
-            topic_difficulties[subtopic] = difficulty.value
-        
-        # Sort subtopics by difficulty
-        return sorted(subtopics, key=lambda x: topic_difficulties[x])
-
-    def _analyze_learning_trends(self, df: pd.DataFrame) -> Dict[str, Any]:
-        """Analyze overall learning trends across the dataset."""
-        return {
-            'sentiment_trend': stats.linregress(
-                range(len(df)),
-                df['sentiment']
-            )._asdict(),
-            'confusion_trend': stats.linregress(
-                range(len(df)),
-                df['shows_confusion']
-            )._asdict(),
-            'engagement_progression': self._calculate_engagement_progression(df)
-        }
-
-    def _calculate_engagement_progression(self, df: pd.DataFrame) -> Dict[str, float]:
-        """Calculate how student engagement changes over time."""
-        df['week'] = df['timestamp'].dt.isocalendar().week
-        weekly_engagement = df.groupby('week').agg({
-            'is_question': 'mean',
-            'shows_confusion': 'mean',
-            'is_followup': 'mean',
-            'sentiment': 'mean'
-        })
-        
-        return {
-            'question_trend': stats.linregress(
-                range(len(weekly_engagement)),
-                weekly_engagement['is_question']
-            ).slope,
-            'confusion_trend': stats.linregress(
-                range(len(weekly_engagement)),
-                weekly_engagement['shows_confusion']
-            ).slope,
-            'follow_up_trend': stats.linregress(
-                range(len(weekly_engagement)),
-                weekly_engagement['is_followup']
-            ).slope,
-            'sentiment_trend': stats.linregress(
-                range(len(weekly_engagement)),
-                weekly_engagement['sentiment']
-            ).slope
-        }
-
-    def _generate_recommendations(self, report: Dict[str, Any]) -> List[str]:
-        """Generate actionable recommendations based on the analysis."""
-        recommendations = []
-        
-        # Analyze difficulty distribution
-        difficult_topics = [
-            topic for topic, data in report['topics'].items()
-            if data['insights'].difficulty_level in 
-            (TopicDifficulty.DIFFICULT, TopicDifficulty.VERY_DIFFICULT)
-        ]
-        
-        if difficult_topics:
-            recommendations.append(
-                f"Consider providing additional resources for challenging topics: {', '.join(difficult_topics)}"
-            )
-        
-        # Analyze student engagement
-        avg_engagement = np.mean([
-            progress['questions_asked'] / progress['total_messages']
-            for progress in report['student_progress'].values()
-        ])
-        
-        if avg_engagement < 0.3:
-            recommendations.append(
-                "Implement more interactive elements to increase student engagement"
-            )
-        
-        # Analyze temporal patterns
-        peak_hours = list(report['temporal_analysis']['peak_activity_hours'].keys())
-        recommendations.append(
-            f"Consider scheduling additional support during peak activity hours: {peak_hours}"
-        )
-        
-        # Analyze learning trends
-        # sentiment_trend = report['temporal_analysis']['learning_trends']['sentiment_trend']
-        # if sentiment_trend < 0:
-        #     recommendations.append(
-        #         "Review teaching approach to address declining student satisfaction"
-        #     )
-        # Analyze learning trends
-        # Analyze learning trends
-        sentiment_trend = report.get('temporal_analysis', {}).get('learning_trends', {}).get('sentiment_trend', None)
-        if isinstance(sentiment_trend, (int, float)):
-            if sentiment_trend < 0:
-                recommendations.append(
-                    "Review teaching approach to address declining student satisfaction"
-                )
-        elif isinstance(sentiment_trend, dict):
-            # Handle the case where sentiment_trend is a dictionary
-            print(f"Unexpected dict format for sentiment_trend: {sentiment_trend}")
-        else:
-            print(f"Unexpected type for sentiment_trend: {type(sentiment_trend)}")
-        
-        return recommendations
-
-class CustomJSONEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, TopicDifficulty):
-            return obj.value
-        if isinstance(obj, TopicInsights):
-            return obj.to_dict()
-        if isinstance(obj, np.integer):
-            return int(obj)
-        if isinstance(obj, np.floating):
-            return float(obj)
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        if isinstance(obj, datetime):
-            return obj.isoformat()
-        return super().default(obj)
-
-def convert_insights_to_dict(report):
-    for main_topic, data in report['topics'].items():
-        if isinstance(data['insights'], TopicInsights):
-            data['insights'] = data['insights'].to_dict()
-        for subtopic, subdata in data['subtopics'].items():
-            if isinstance(subdata['insights'], TopicInsights):
-                subdata['insights'] = subdata['insights'].to_dict()
-
-if __name__ == "__main__":
-    # Load chat history data
-    chat_history = None
-    with open('sample_files/chat_history_corpus.json', 'r', encoding="utf-8") as file:
-        chat_history = json.load(file)
-    
-    # Initialize analytics system
-    analytics = PreClassAnalytics()
-    
-    # Generate comprehensive report
-    report = analytics.generate_comprehensive_report(chat_history)
-
-    # Convert insights to dictionary
-    # convert_insights_to_dict(report)
-    
-    print(json.dumps(report, indent=4, cls=CustomJSONEncoder))
+import pandas as pd
+import numpy as np
+from datetime import datetime
+from typing import List, Dict, Any, Tuple
+import spacy
+from collections import Counter, defaultdict
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from textblob import TextBlob
+import networkx as nx
+from scipy import stats
+import logging
+import json
+from dataclasses import dataclass
+from enum import Enum
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class TopicDifficulty(Enum):
+    EASY = "easy"
+    MODERATE = "moderate"
+    DIFFICULT = "difficult"
+    VERY_DIFFICULT = "very_difficult"
+
+
+@dataclass
+class QuestionMetrics:
+    complexity_score: float
+    follow_up_count: int
+    clarification_count: int
+    time_spent: float
+    sentiment_score: float
+
+@dataclass
+class TopicInsights:
+    difficulty_level: TopicDifficulty
+    common_confusion_points: List[str]
+    question_patterns: List[str]
+    time_distribution: Dict[str, float]
+    engagement_metrics: Dict[str, float]
+    recommended_focus_areas: List[str]
+
+    def to_dict(self):
+        return {
+            "difficulty_level": self.difficulty_level.value,  # Convert enum to its value
+            "common_confusion_points": self.common_confusion_points,
+            "question_patterns": self.question_patterns,
+            "time_distribution": {str(k): v for k, v in self.time_distribution.items()},
+            "engagement_metrics": self.engagement_metrics,
+            "recommended_focus_areas": self.recommended_focus_areas,
+        }
+
+class PreClassAnalytics:
+    def __init__(self, nlp_model: str = "en_core_web_lg"):
+        """Initialize the analytics system with necessary components."""
+        self.nlp = spacy.load(nlp_model)
+        self.question_indicators = {
+            "what", "why", "how", "when", "where", "which", "who", 
+            "whose", "whom", "can", "could", "would", "will", "explain"
+        }
+        self.confusion_indicators = {
+            "confused", "don't understand", "unclear", "not clear",
+            "stuck", "difficult", "hard", "help", "explain again"
+        }
+        self.follow_up_indicators = {
+            "also", "another", "additionally", "furthermore", "moreover",
+            "besides", "related", "similarly", "again"
+        }
+        
+    def preprocess_chat_history(self, chat_history: List[Dict]) -> pd.DataFrame:
+        """Convert chat history to DataFrame with enhanced features."""
+        messages = []
+        for chat in chat_history:
+            user_id = chat['user_id']['$oid']
+            for msg in chat['messages']:
+                try:
+                    # Ensure the timestamp is in the correct format
+                    if isinstance(msg['timestamp'], dict) and '$date' in msg['timestamp']:
+                        timestamp = pd.to_datetime(msg['timestamp']['$date'])
+                    elif isinstance(msg['timestamp'], str):
+                        timestamp = pd.to_datetime(msg['timestamp'])
+                    else:
+                        raise ValueError("Invalid timestamp format")
+                except Exception as e:
+                    print(f"Error parsing timestamp: {msg['timestamp']}, error: {e}")
+                    timestamp = pd.NaT  # Use NaT (Not a Time) for invalid timestamps
+                
+                messages.append({
+                    'user_id': user_id,
+                    'timestamp': timestamp,
+                    'prompt': msg['prompt'],
+                    'response': msg['response'],
+                    'is_question': any(q in msg['prompt'].lower() for q in self.question_indicators),
+                    'shows_confusion': any(c in msg['prompt'].lower() for c in self.confusion_indicators),
+                    'is_followup': any(f in msg['prompt'].lower() for f in self.follow_up_indicators)
+                })
+        
+        df = pd.DataFrame(messages)
+        df['sentiment'] = df['prompt'].apply(lambda x: TextBlob(x).sentiment.polarity)
+        return df
+
+    def extract_topic_hierarchies(self, df: pd.DataFrame) -> Dict[str, List[str]]:
+        """Extract hierarchical topic relationships from conversations."""
+        topic_hierarchy = defaultdict(list)
+        
+        for _, row in df.iterrows():
+            doc = self.nlp(row['prompt'])
+            
+            # Extract main topics and subtopics using noun chunks and dependencies
+            main_topics = []
+            subtopics = []
+            
+            for chunk in doc.noun_chunks:
+                if chunk.root.dep_ in ('nsubj', 'dobj'):
+                    main_topics.append(chunk.text.lower())
+                else:
+                    subtopics.append(chunk.text.lower())
+            
+            # Build hierarchy
+            for main_topic in main_topics:
+                topic_hierarchy[main_topic].extend(subtopics)
+        
+        # Clean and deduplicate
+        return {k: list(set(v)) for k, v in topic_hierarchy.items()}
+
+    def analyze_topic_difficulty(self, df: pd.DataFrame, topic: str) -> TopicDifficulty:
+        """Determine topic difficulty based on various metrics."""
+        topic_msgs = df[df['prompt'].str.contains(topic, case=False)]
+        
+        # Calculate difficulty indicators
+        confusion_rate = topic_msgs['shows_confusion'].mean()
+        question_rate = topic_msgs['is_question'].mean()
+        follow_up_rate = topic_msgs['is_followup'].mean()
+        avg_sentiment = topic_msgs['sentiment'].mean()
+        
+        # Calculate composite difficulty score
+        difficulty_score = (
+            confusion_rate * 0.4 +
+            question_rate * 0.3 +
+            follow_up_rate * 0.2 +
+            (1 - (avg_sentiment + 1) / 2) * 0.1
+        )
+        
+        # Map score to difficulty level
+        if difficulty_score < 0.3:
+            return TopicDifficulty.EASY
+        elif difficulty_score < 0.5:
+            return TopicDifficulty.MODERATE
+        elif difficulty_score < 0.7:
+            return TopicDifficulty.DIFFICULT
+        else:
+            return TopicDifficulty.VERY_DIFFICULT
+
+    def identify_confusion_patterns(self, df: pd.DataFrame, topic: str) -> List[str]:
+        """Identify common patterns in student confusion."""
+        confused_msgs = df[
+            (df['prompt'].str.contains(topic, case=False)) & 
+            (df['shows_confusion'])
+        ]['prompt']
+        
+        patterns = []
+        for msg in confused_msgs:
+            doc = self.nlp(msg)
+            
+            # Extract key phrases around confusion indicators
+            for sent in doc.sents:
+                for token in sent:
+                    if token.text.lower() in self.confusion_indicators:
+                        # Get context window around confusion indicator
+                        context = sent.text
+                        patterns.append(context)
+        
+        # Group similar patterns
+        if patterns:
+            vectorizer = TfidfVectorizer(ngram_range=(1, 3))
+            tfidf_matrix = vectorizer.fit_transform(patterns)
+            similarity_matrix = cosine_similarity(tfidf_matrix)
+            
+            # Cluster similar patterns
+            G = nx.Graph()
+            for i in range(len(patterns)):
+                for j in range(i + 1, len(patterns)):
+                    if similarity_matrix[i][j] > 0.5:  # Similarity threshold
+                        G.add_edge(i, j)
+            
+            # Extract representative patterns from each cluster
+            clusters = list(nx.connected_components(G))
+            return [patterns[min(cluster)] for cluster in clusters]
+        
+        return []
+
+    def analyze_question_patterns(self, df: pd.DataFrame, topic: str) -> List[str]:
+        """Analyze patterns in student questions about the topic."""
+        topic_questions = df[
+            (df['prompt'].str.contains(topic, case=False)) & 
+            (df['is_question'])
+        ]['prompt']
+        
+        question_types = defaultdict(list)
+        for question in topic_questions:
+            doc = self.nlp(question)
+            
+            # Categorize questions
+            if any(token.text.lower() in {"what", "define", "explain"} for token in doc):
+                question_types["conceptual"].append(question)
+            elif any(token.text.lower() in {"how", "steps", "process"} for token in doc):
+                question_types["procedural"].append(question)
+            elif any(token.text.lower() in {"why", "reason", "because"} for token in doc):
+                question_types["reasoning"].append(question)
+            else:
+                question_types["other"].append(question)
+        
+        # Extract patterns from each category
+        patterns = []
+        for category, questions in question_types.items():
+            if questions:
+                vectorizer = TfidfVectorizer(ngram_range=(1, 3))
+                tfidf_matrix = vectorizer.fit_transform(questions)
+                
+                # Get most representative questions
+                feature_array = np.mean(tfidf_matrix.toarray(), axis=0)
+                tfidf_sorting = np.argsort(feature_array)[::-1]
+                features = vectorizer.get_feature_names_out()
+                
+                patterns.append(f"{category}: {' '.join(features[tfidf_sorting[:3]])}")
+        
+        return patterns
+
+    def analyze_time_distribution(self, df: pd.DataFrame, topic: str) -> Dict[str, float]:
+        """Analyze time spent on different aspects of the topic."""
+        topic_msgs = df[df['prompt'].str.contains(topic, case=False)].copy()
+        if len(topic_msgs) < 2:
+            return {}
+        
+        topic_msgs['time_diff'] = topic_msgs['timestamp'].diff()
+        
+        # Calculate time distribution
+        distribution = {
+            'total_time': topic_msgs['time_diff'].sum().total_seconds() / 60,
+            'avg_time_per_message': topic_msgs['time_diff'].mean().total_seconds() / 60,
+            'max_time_gap': topic_msgs['time_diff'].max().total_seconds() / 60,
+            'time_spent_on_questions': topic_msgs[topic_msgs['is_question']]['time_diff'].sum().total_seconds() / 60,
+            'time_spent_on_confusion': topic_msgs[topic_msgs['shows_confusion']]['time_diff'].sum().total_seconds() / 60
+        }
+        
+        return distribution
+
+    def calculate_engagement_metrics(self, df: pd.DataFrame, topic: str) -> Dict[str, float]:
+        """Calculate student engagement metrics for the topic."""
+        topic_msgs = df[df['prompt'].str.contains(topic, case=False)]
+        
+        metrics = {
+            'message_count': len(topic_msgs),
+            'question_ratio': topic_msgs['is_question'].mean(),
+            'confusion_ratio': topic_msgs['shows_confusion'].mean(),
+            'follow_up_ratio': topic_msgs['is_followup'].mean(),
+            'avg_sentiment': topic_msgs['sentiment'].mean(),
+            'engagement_score': 0.0  # Will be calculated below
+        }
+        
+        # Calculate engagement score
+        metrics['engagement_score'] = (
+            metrics['message_count'] * 0.3 +
+            metrics['question_ratio'] * 0.25 +
+            metrics['follow_up_ratio'] * 0.25 +
+            (metrics['avg_sentiment'] + 1) / 2 * 0.2  # Normalize sentiment to 0-1
+        )
+        
+        return metrics
+
+    def generate_topic_insights(self, df: pd.DataFrame, topic: str) -> TopicInsights:
+        """Generate comprehensive insights for a topic."""
+        difficulty = self.analyze_topic_difficulty(df, topic)
+        confusion_points = self.identify_confusion_patterns(df, topic)
+        question_patterns = self.analyze_question_patterns(df, topic)
+        time_distribution = self.analyze_time_distribution(df, topic)
+        engagement_metrics = self.calculate_engagement_metrics(df, topic)
+        
+        # Generate recommended focus areas based on insights
+        focus_areas = []
+        
+        if difficulty in (TopicDifficulty.DIFFICULT, TopicDifficulty.VERY_DIFFICULT):
+            focus_areas.append("Fundamental concept reinforcement needed")
+        
+        if confusion_points:
+            focus_areas.append(f"Address common confusion around: {', '.join(confusion_points[:3])}")
+        
+        if engagement_metrics['confusion_ratio'] > 0.3:
+            focus_areas.append("Consider alternative teaching approaches")
+        
+        if time_distribution.get('time_spent_on_questions', 0) > time_distribution.get('total_time', 0) * 0.5:
+            focus_areas.append("More practical examples or demonstrations needed")
+        
+        return TopicInsights(
+            difficulty_level=difficulty,
+            common_confusion_points=confusion_points,
+            question_patterns=question_patterns,
+            time_distribution=time_distribution,
+            engagement_metrics=engagement_metrics,
+            recommended_focus_areas=focus_areas
+        )
+
+    def analyze_student_progress(self, df: pd.DataFrame) -> Dict[str, Any]:
+        """Analyze individual student progress and learning patterns."""
+        student_progress = {}
+        
+        for student_id in df['user_id'].unique():
+            student_msgs = df[df['user_id'] == student_id]
+            
+            # Calculate student-specific metrics
+            progress = {
+                'total_messages': len(student_msgs),
+                'questions_asked': student_msgs['is_question'].sum(),
+                'confusion_instances': student_msgs['shows_confusion'].sum(),
+                'avg_sentiment': student_msgs['sentiment'].mean(),
+                'topic_engagement': {},
+                'learning_pattern': self._identify_learning_pattern(student_msgs)
+            }
+            
+            # Analyze topic-specific engagement
+            topics = self.extract_topic_hierarchies(student_msgs)
+            for topic in topics:
+                topic_msgs = student_msgs[student_msgs['prompt'].str.contains(topic, case=False)]
+                progress['topic_engagement'][topic] = {
+                    'message_count': len(topic_msgs),
+                    'confusion_rate': topic_msgs['shows_confusion'].mean(),
+                    'sentiment_trend': stats.linregress(
+                        range(len(topic_msgs)),
+                        topic_msgs['sentiment']
+                    ).slope
+                }
+            
+            student_progress[student_id] = progress
+        
+        return student_progress
+
+    def _identify_learning_pattern(self, student_msgs: pd.DataFrame) -> str:
+        """Identify student's learning pattern based on their interaction style."""
+        # Calculate key metrics
+        question_ratio = student_msgs['is_question'].mean()
+        confusion_ratio = student_msgs['shows_confusion'].mean()
+        follow_up_ratio = student_msgs['is_followup'].mean()
+        sentiment_trend = stats.linregress(
+            range(len(student_msgs)),
+            student_msgs['sentiment']
+        ).slope
+        
+        # Identify pattern
+        if question_ratio > 0.6:
+            return "Inquisitive Learner"
+        elif confusion_ratio > 0.4:
+            return "Needs Additional Support"
+        elif follow_up_ratio > 0.5:
+            return "Deep Dive Learner"
+        elif sentiment_trend > 0:
+            return "Progressive Learner"
+        else:
+            return "Steady Learner"
+
+    def generate_comprehensive_report(self, chat_history: List[Dict]) -> Dict[str, Any]:
+        """Generate a comprehensive analytics report."""
+        # Preprocess chat history
+        df = self.preprocess_chat_history(chat_history)
+        
+        # Extract topics
+        topics = self.extract_topic_hierarchies(df)
+        
+        report = {
+            'topics': {},
+            'student_progress': self.analyze_student_progress(df),
+            'overall_metrics': {
+                'total_conversations': len(df),
+                'unique_students': df['user_id'].nunique(),
+                'avg_sentiment': df['sentiment'].mean(),
+                'most_discussed_topics': Counter(
+                    topic for topics_list in topics.values() 
+                    for topic in topics_list
+                ).most_common(5)
+            }
+        }
+        
+        # Generate topic-specific insights
+        for main_topic, subtopics in topics.items():
+            subtopic_insights = {}
+            for subtopic in subtopics:
+                subtopic_insights[subtopic] = {
+                    'insights': self.generate_topic_insights(df, subtopic),
+                    'related_topics': [t for t in subtopics if t != subtopic],
+                    'student_engagement': {
+                        student_id: self.calculate_engagement_metrics(
+                            df[df['user_id'] == student_id], 
+                            subtopic
+                        )
+                        for student_id in df['user_id'].unique()
+                    }
+                }
+            
+            report['topics'][main_topic] = {
+                'insights': self.generate_topic_insights(df, main_topic),
+                'subtopics': subtopic_insights,
+                'topic_relationships': {
+                    'hierarchy_depth': len(subtopics),
+                    'connection_strength': self._calculate_topic_connections(df, main_topic, subtopics),
+                    'progression_path': self._identify_topic_progression(df, main_topic, subtopics)
+                }
+            }
+        
+        # Add temporal analysis
+        report['temporal_analysis'] = {
+            'daily_engagement': df.groupby(df['timestamp'].dt.date).agg({
+                'user_id': 'count',
+                'is_question': 'sum',
+                'shows_confusion': 'sum',
+                'sentiment': 'mean'
+            }).to_dict(),
+            'peak_activity_hours': df.groupby(df['timestamp'].dt.hour)['user_id'].count().nlargest(3).to_dict(),
+            'learning_trends': self._analyze_learning_trends(df)
+        }
+        
+        # Add recommendations
+        report['recommendations'] = self._generate_recommendations(report)
+        
+        return report
+
+    def _calculate_topic_connections(self, df: pd.DataFrame, main_topic: str, subtopics: List[str]) -> Dict[str, float]:
+        """Calculate connection strength between topics based on co-occurrence."""
+        connections = {}
+        main_topic_msgs = df[df['prompt'].str.contains(main_topic, case=False)]
+        
+        for subtopic in subtopics:
+            cooccurrence = df[
+                df['prompt'].str.contains(main_topic, case=False) & 
+                df['prompt'].str.contains(subtopic, case=False)
+            ].shape[0]
+            
+            connection_strength = cooccurrence / len(main_topic_msgs) if len(main_topic_msgs) > 0 else 0
+            connections[subtopic] = connection_strength
+        
+        return connections
+
+    def _identify_topic_progression(self, df: pd.DataFrame, main_topic: str, subtopics: List[str]) -> List[str]:
+        """Identify optimal topic progression path based on student interactions."""
+        topic_difficulties = {}
+        
+        for subtopic in subtopics:
+            difficulty = self.analyze_topic_difficulty(df, subtopic)
+            topic_difficulties[subtopic] = difficulty.value
+        
+        # Sort subtopics by difficulty
+        return sorted(subtopics, key=lambda x: topic_difficulties[x])
+
+    def _analyze_learning_trends(self, df: pd.DataFrame) -> Dict[str, Any]:
+        """Analyze overall learning trends across the dataset."""
+        return {
+            'sentiment_trend': stats.linregress(
+                range(len(df)),
+                df['sentiment']
+            )._asdict(),
+            'confusion_trend': stats.linregress(
+                range(len(df)),
+                df['shows_confusion']
+            )._asdict(),
+            'engagement_progression': self._calculate_engagement_progression(df)
+        }
+
+    def _calculate_engagement_progression(self, df: pd.DataFrame) -> Dict[str, float]:
+        """Calculate how student engagement changes over time."""
+        df['week'] = df['timestamp'].dt.isocalendar().week
+        weekly_engagement = df.groupby('week').agg({
+            'is_question': 'mean',
+            'shows_confusion': 'mean',
+            'is_followup': 'mean',
+            'sentiment': 'mean'
+        })
+        
+        return {
+            'question_trend': stats.linregress(
+                range(len(weekly_engagement)),
+                weekly_engagement['is_question']
+            ).slope,
+            'confusion_trend': stats.linregress(
+                range(len(weekly_engagement)),
+                weekly_engagement['shows_confusion']
+            ).slope,
+            'follow_up_trend': stats.linregress(
+                range(len(weekly_engagement)),
+                weekly_engagement['is_followup']
+            ).slope,
+            'sentiment_trend': stats.linregress(
+                range(len(weekly_engagement)),
+                weekly_engagement['sentiment']
+            ).slope
+        }
+
+    def _generate_recommendations(self, report: Dict[str, Any]) -> List[str]:
+        """Generate actionable recommendations based on the analysis."""
+        recommendations = []
+        
+        # Analyze difficulty distribution
+        difficult_topics = [
+            topic for topic, data in report['topics'].items()
+            if data['insights'].difficulty_level in 
+            (TopicDifficulty.DIFFICULT, TopicDifficulty.VERY_DIFFICULT)
+        ]
+        
+        if difficult_topics:
+            recommendations.append(
+                f"Consider providing additional resources for challenging topics: {', '.join(difficult_topics)}"
+            )
+        
+        # Analyze student engagement
+        avg_engagement = np.mean([
+            progress['questions_asked'] / progress['total_messages']
+            for progress in report['student_progress'].values()
+        ])
+        
+        if avg_engagement < 0.3:
+            recommendations.append(
+                "Implement more interactive elements to increase student engagement"
+            )
+        
+        # Analyze temporal patterns
+        peak_hours = list(report['temporal_analysis']['peak_activity_hours'].keys())
+        recommendations.append(
+            f"Consider scheduling additional support during peak activity hours: {peak_hours}"
+        )
+        
+        # Analyze learning trends
+        # sentiment_trend = report['temporal_analysis']['learning_trends']['sentiment_trend']
+        # if sentiment_trend < 0:
+        #     recommendations.append(
+        #         "Review teaching approach to address declining student satisfaction"
+        #     )
+        # Analyze learning trends
+        # Analyze learning trends
+        sentiment_trend = report.get('temporal_analysis', {}).get('learning_trends', {}).get('sentiment_trend', None)
+        if isinstance(sentiment_trend, (int, float)):
+            if sentiment_trend < 0:
+                recommendations.append(
+                    "Review teaching approach to address declining student satisfaction"
+                )
+        elif isinstance(sentiment_trend, dict):
+            # Handle the case where sentiment_trend is a dictionary
+            print(f"Unexpected dict format for sentiment_trend: {sentiment_trend}")
+        else:
+            print(f"Unexpected type for sentiment_trend: {type(sentiment_trend)}")
+        
+        return recommendations
+
+class CustomJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, TopicDifficulty):
+            return obj.value
+        if isinstance(obj, TopicInsights):
+            return obj.to_dict()
+        if isinstance(obj, np.integer):
+            return int(obj)
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        if isinstance(obj, datetime):
+            return obj.isoformat()
+        return super().default(obj)
+
+def convert_insights_to_dict(report):
+    for main_topic, data in report['topics'].items():
+        if isinstance(data['insights'], TopicInsights):
+            data['insights'] = data['insights'].to_dict()
+        for subtopic, subdata in data['subtopics'].items():
+            if isinstance(subdata['insights'], TopicInsights):
+                subdata['insights'] = subdata['insights'].to_dict()
+
+if __name__ == "__main__":
+    # Load chat history data
+    chat_history = None
+    with open('sample_files/chat_history_corpus.json', 'r', encoding="utf-8") as file:
+        chat_history = json.load(file)
+    
+    # Initialize analytics system
+    analytics = PreClassAnalytics()
+    
+    # Generate comprehensive report
+    report = analytics.generate_comprehensive_report(chat_history)
+
+    # Convert insights to dictionary
+    # convert_insights_to_dict(report)
+    
+    print(json.dumps(report, indent=4, cls=CustomJSONEncoder))
     # print(report)

requirements.txt

@@ -1,31 +1,36 @@
-streamlit
-pymongo
-PyPDF2
-python-docx
-openai
-google-generativeai
-llama-index
-werkzeug
-numpy
-pandas
-plotly
-scikit-learn
-networkx
-community
-umap-learn
-seaborn
-matplotlib
-scipy
-Pillow
-python-dotenv
-zoomus
-asyncio
-google-auth-oauthlib
-google-auth
-transformers
-textstat
-spacy
-streamlit_option_menu
-beautifulsoup4
-youtube-transcript-api
-requests
+streamlit
+pymongo
+PyPDF2
+python-docx
+openai
+google-generativeai
+llama-index
+werkzeug
+numpy
+pandas
+plotly
+scikit-learn
+networkx
+community
+umap-learn
+seaborn
+matplotlib
+scipy
+Pillow
+python-dotenv
+zoomus
+asyncio
+google-auth-oauthlib
+google-auth
+transformers
+textstat
+spacy
+streamlit_option_menu
+beautifulsoup4
+youtube-transcript-api
+requests
+xml==0.0.1
+networkx==3.1
+bokeh==3.2.1
+scikit-learn==1.2.2
+langchain==0.0.208

research22.py

@@ -0,0 +1,517 @@
+# if __name__ == "__main__":
+#     main()
+import streamlit as st
+import google.generativeai as genai
+from typing import Dict, Any
+import PyPDF2
+import io
+from pymongo import MongoClient
+from dotenv import load_dotenv
+import os
+import json
+import re
+
+# --------------------------------------------------------------------------------
+# 1. Environment Setup
+# --------------------------------------------------------------------------------
+load_dotenv()
+# MongoDB
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+# Gemini
+GEMINI_KEY = os.getenv("GEMINI_KEY", "AIzaSyCFIvntck54HOCS5pxxiy9wpr5HJN3r02I")
+
+# Configure Gemini
+genai.configure(api_key=GEMINI_KEY)
+
+
+# --------------------------------------------------------------------------------
+# 2. Database Connection
+# --------------------------------------------------------------------------------
+def create_db_connection():
+    """
+    Create MongoDB connection and return the 'papers' collection.
+    """
+    try:
+        client = MongoClient(MONGODB_URI)
+        db = client["novascholar_db"]  # Database name
+        collection = db["research_papers"]  # Collection name
+        # Ping to confirm connection
+        client.admin.command("ping")
+        return db
+    except Exception as e:
+        st.error(f"Database connection error: {str(e)}")
+        return None
+
+
+# --------------------------------------------------------------------------------
+# 3. PDF Text Extraction
+# --------------------------------------------------------------------------------
+def extract_text_from_pdf(pdf_file) -> str:
+    """
+    Extract all text from a PDF.
+    """
+    try:
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        text = ""
+        for page in pdf_reader.pages:
+            text += page.extract_text() + "\n"
+        return text
+    except Exception as e:
+        st.error(f"Error processing PDF: {str(e)}")
+        return ""
+
+
+# --------------------------------------------------------------------------------
+# 4. Gemini Response Helper
+# --------------------------------------------------------------------------------
+def get_gemini_response(prompt: str) -> str:
+    """
+    Sends a prompt to Google's Gemini model and returns the response text.
+    Adjust this function as needed for your generative AI usage.
+    """
+    try:
+        model = genai.GenerativeModel("gemini-pro")
+        response = model.generate_content(prompt)
+        return response.text
+    except Exception as e:
+        st.error(f"Gemini API Error: {str(e)}")
+        return ""
+
+
+# --------------------------------------------------------------------------------
+# 5. Basic Info Extraction
+# --------------------------------------------------------------------------------
+def extract_basic_info(text: str) -> Dict[str, str]:
+    """
+    Extract title, publication, journal/conference, abstract, keywords, author, and date from the paper text.
+    Return a dictionary with these fields.
+    """
+    prompt = f"""
+    Extract the following fields from the research paper text below:
+
+    Title
+    Publication
+    Journal_Conference
+    Abstract
+    Keywords
+    Author
+    Date_of_Publication
+
+    Paper text:
+    {text}
+
+    Return them in this format:
+    Title: ...
+    Publication: ...
+    Journal_Conference: ...
+    Abstract: ...
+    Keywords: ...
+    Author: ...
+    Date_of_Publication: ...
+    """
+    response = get_gemini_response(prompt)
+    if not response:
+        return {}
+    info = {}
+    lines = response.split("\n")
+    for line in lines:
+        if ":" in line:
+            key, value = line.split(":", 1)
+            info[key.strip()] = value.strip()
+    return info
+
+
+# --------------------------------------------------------------------------------
+# 6. Content Sections Extraction
+# --------------------------------------------------------------------------------
+def extract_content_sections(text: str) -> Dict[str, str]:
+    """
+    Extract expanded sections: Intro, Literature_Review, Research_Models_Used,
+    Methodology, Discussion, Future_Scope, Theory.
+    """
+    prompt = f"""Please extract these sections from the research paper:
+    1. Introduction
+    2. Literature Review
+    3. Research Models Used
+    4. Methodology
+    5. Discussion
+    6. Future Scope
+    7. Theory
+
+    Paper text: {text}
+    
+    Return in this exact format without any additional text or explanations also make sure 
+    no data should be empty (at least 10-15 words) and it should be meaningful:
+    Intro: <text>
+    Literature_Review: <text>
+    Research_Models_Used: <text>
+    Methodology: <text>
+    Discussion: <text>
+    Future_Scope: <text>
+    Theory: <text>
+    """
+    response = get_gemini_response(prompt)
+    if not response:
+        return {}
+    sections = {}
+    lines = response.split("\n")
+    for line in lines:
+        if ":" in line:
+            key, value = line.split(":", 1)
+            sections[key.strip()] = value.strip()
+    return sections
+
+
+# --------------------------------------------------------------------------------
+# 7. Variables Extraction
+# --------------------------------------------------------------------------------
+def extract_variables(text: str) -> Dict[str, Any]:
+    """
+    Extract variable data: Independent_Variables, nof_Independent_Variables,
+    Dependent_Variables, nof_Dependent_Variables, Control_Variables,
+    Extraneous_Variables, nof_Control_Variables, nof_Extraneous_Variables
+    """
+    prompt = f"""From the paper text, extract the following fields:
+    1. Independent_Variables
+    2. nof_Independent_Variables
+    3. Dependent_Variables
+    4. nof_Dependent_Variables
+    5. Control_Variables
+    6. Extraneous_Variables
+    7. nof_Control_Variables
+    8. nof_Extraneous_Variables
+
+    Return them in this format:
+    Independent_Variables: <list>
+    nof_Independent_Variables: <integer>
+    Dependent_Variables: <list>
+    nof_Dependent_Variables: <integer>
+    Control_Variables: <list>
+    Extraneous_Variables: <list>
+    nof_Control_Variables: <integer>
+    nof_Extraneous_Variables: <integer>
+
+    Paper text: {text}
+    """
+    response = get_gemini_response(prompt)
+    if not response:
+        return {}
+    variables = {}
+    lines = response.split("\n")
+    for line in lines:
+        if ":" in line:
+            key, value = line.split(":", 1)
+            # Attempt to convert to integer where appropriate
+            clean_key = key.strip()
+            clean_value = value.strip()
+            if clean_key.startswith("nof_"):
+                try:
+                    variables[clean_key] = int(clean_value)
+                except ValueError:
+                    # fallback if it's not an integer
+                    variables[clean_key] = 0
+            else:
+                variables[clean_key] = clean_value
+    return variables
+
+
+# --------------------------------------------------------------------------------
+# 8. Utility to ensure no empty fields (example logic)
+# --------------------------------------------------------------------------------
+def ensure_non_empty_values(data: Dict[str, Any], fallback_text: str) -> Dict[str, Any]:
+    """
+    Ensure each extracted field has meaningful content. If empty, fill with default text.
+    """
+    for k, v in data.items():
+        if not v or len(str(v).split()) < 3:  # example check for minimal words
+            data[k] = f"No sufficient data found for {k}. Could not parse."
+    return data
+
+
+# --------------------------------------------------------------------------------
+# 9. Processing the Paper
+# --------------------------------------------------------------------------------
+# def process_paper(text: str) -> Dict[str, Any]:
+#     """
+#     Orchestrate calls to extract basic info, content sections, and variables.
+#     Return a dictionary containing all the fields with consistent naming.
+#     """
+#     with st.spinner("Extracting basic information..."):
+#         basic_info = extract_basic_info(text)
+#         basic_info = ensure_non_empty_values(basic_info, text)
+
+#     with st.spinner("Extracting content sections..."):
+#         content_sections = extract_content_sections(text)
+#         content_sections = ensure_non_empty_values(content_sections, text)
+
+#     with st.spinner("Extracting variables..."):
+#         variables_info = extract_variables(text)
+#         variables_info = ensure_non_empty_values(variables_info, text)
+
+#     # Create a single dictionary with all fields
+#     paper_doc = {
+#         "Title": basic_info.get("Title", ""),
+#         "Publication": basic_info.get("Publication", ""),
+#         "Journal_Conference": basic_info.get("Journal_Conference", ""),
+#         "Abstract": basic_info.get("Abstract", ""),
+#         "Keywords": basic_info.get("Keywords", ""),
+#         "Author": basic_info.get("Author", ""),
+#         "Date_of_Publication": basic_info.get("Date_of_Publication", ""),
+#         "Intro": content_sections.get("Intro", ""),
+#         "Literature_Review": content_sections.get("Literature_Review", ""),
+#         "Research_Models_Used": content_sections.get("Research_Models_Used", ""),
+#         "Methodology": content_sections.get("Methodology", ""),
+#         "Discussion": content_sections.get("Discussion", ""),
+#         "Future_Scope": content_sections.get("Future_Scope", ""),
+#         "Theory": content_sections.get("Theory", ""),
+#         "Independent_Variables": variables_info.get("Independent_Variables", ""),
+#         "nof_Independent_Variables": variables_info.get("nof_Independent_Variables", 0),
+#         "Dependent_Variables": variables_info.get("Dependent_Variables", ""),
+#         "nof_Dependent_Variables": variables_info.get("nof_Dependent_Variables", 0),
+#         "Control_Variables": variables_info.get("Control_Variables", ""),
+#         "Extraneous_Variables": variables_info.get("Extraneous_Variables", ""),
+#         "nof_Control_Variables": variables_info.get("nof_Control_Variables", 0),
+#         "nof_Extraneous_Variables": variables_info.get("nof_Extraneous_Variables", 0),
+#     }
+
+#     return paper_doc
+
+# filepath: /c:/Users/acer/OneDrive/Documents/GitHub/res-cor/research22.py
+# ...existing code continues...
+
+# --------------------------------------------------------------------------------
+# 3. Paper Type Attributes
+# --------------------------------------------------------------------------------
+PAPER_TYPE_ATTRIBUTES = {
+    "Review Based Paper": [
+        "Title",
+        "Publication",
+        "Journal_Conference",
+        "Abstract",
+        "Keywords",
+        "Author",
+        "Date_of_Publication",
+        "Intro",
+        "Literature_Review",
+        "Body",
+        "Protocol",
+        "Search String",
+        "Included Studies",
+        "Data Collection and Analysis Methods",
+        "Data Extraction Table",
+        "Synthesis and Analysis",
+        "Conclusion",
+        "Limitations",
+        "Results",
+        "References",
+        "Risk of Bias Assessment",
+    ],
+    "Opinion/Perspective Based Paper": [
+        "Title",
+        "Publication",
+        "Journal_Conference",
+        "Abstract",
+        "Keywords",
+        "Author",
+        "Date_of_Publication",
+        "Intro",
+        "Literature_Review",
+        "Introduction",
+        "Body",
+        "Results and Discussion",
+        "Conclusion",
+        "References",
+    ],
+    "Empirical Research Paper": [
+        "Title",
+        "Publication",
+        "Journal_Conference",
+        "Abstract",
+        "Keywords",
+        "Author",
+        "Date_of_Publication",
+        "Intro",
+        "Literature_Review",
+        "Introduction",
+        "Body",
+        "Methodology",
+        "Participants",
+        "Survey Instrument",
+        "Data Collection",
+        "Data Analysis",
+        "Results and Discussion",
+        "Conclusion",
+        "References",
+    ],
+    "Research Paper (Other)": [
+        "Title",
+        "Publication",
+        "Journal_Conference",
+        "Abstract",
+        "Keywords",
+        "Author",
+        "Date_of_Publication",
+        "Intro",
+        "Literature_Review",
+        "Research_Models_Used",
+        "Methodology",
+        "Discussion",
+        "Future_Scope",
+        "Theory",
+        "Independent_Variables",
+        "nof_Independent_Variables",
+        "Dependent_Variables",
+        "nof_Dependent_Variables",
+        "Control_Variables",
+        "Extraneous_Variables",
+        "nof_Control_Variables",
+        "nof_Extraneous_Variables",
+    ],
+}
+
+
+# --------------------------------------------------------------------------------
+# 4. Extract Paper Fields
+# --------------------------------------------------------------------------------
+def extract_paper_fields(text: str, paper_type: str) -> Dict[str, Any]:
+    """
+    Use Gemini to extract fields based on the paper type attributes,
+    then return a dictionary of extracted fields.
+    """
+    if paper_type not in PAPER_TYPE_ATTRIBUTES:
+        st.error("Invalid paper type selected.")
+        return {}
+
+    selected_attrs = PAPER_TYPE_ATTRIBUTES[paper_type]
+    prompt = f"""
+    Extract the following fields from the research paper text below:
+
+    {", ".join(selected_attrs)}
+
+    Paper text:
+    {text}
+
+    Return them in this JSON format strictly, with no extra text:
+    [
+        {{
+            {", ".join([f'"{attr}": "value"' for attr in selected_attrs])}
+        }}
+    ]
+    """
+
+    try:
+        response = get_gemini_response(prompt)
+        if not response:
+            st.error("No response from Gemini.")
+            return {}
+
+        # Clean up any text around JSON
+        # Clean up any text around JSON
+        raw_text = response.strip()
+
+        # Find start and end of JSON
+        json_start = raw_text.find("[")
+        json_end = raw_text.rfind("]") + 1
+        json_str = raw_text[json_start:json_end]
+
+        # Try removing trailing commas, extra quotes, etc.
+        json_str = re.sub(r",\s*}", "}", json_str)
+        json_str = re.sub(r",\s*\]", "]", json_str)
+
+        try:
+            data = json.loads(json_str)
+        except json.JSONDecodeError as e:
+            st.warning(f"Fixing JSON errors: {str(e)}")
+            # As a last-resort attempt, remove anything after the last curly bracket
+            bracket_pos = json_str.rfind("}")
+            if bracket_pos != -1:
+                json_str = json_str[: bracket_pos + 1]
+            # Try again
+            data = json.loads(json_str)
+
+        if isinstance(data, list) and len(data) > 0:
+            return data[0]
+        else:
+            st.error("Gemini did not return a valid JSON array.")
+            return {}
+    except Exception as e:
+        st.error(f"Error in Gemini extraction: {str(e)}")
+        return {}
+
+
+# --------------------------------------------------------------------------------
+# 5. Process Paper and Save
+# --------------------------------------------------------------------------------
+def process_paper(text: str, paper_type: str):
+    """
+    Extract paper fields based on paper type, then save to
+    the corresponding MongoDB collection.
+    """
+    db = create_db_connection()
+    if not db:
+        return
+
+    # Determine collection name
+    collection_name = paper_type.replace(" ", "_").lower()
+    collection = db[collection_name]
+
+    # Extract fields
+    extracted_data = extract_paper_fields(text, paper_type)
+    if extracted_data:
+        # Insert into MongoDB
+        collection.insert_one(extracted_data)
+        return extracted_data
+    return {}
+
+
+# --------------------------------------------------------------------------------
+# 6. Streamlit UI for Paper Extraction
+# --------------------------------------------------------------------------------
+def main():
+    # st.set_page_config(page_title="Extract Research Paper", layout="wide")
+    st.title("Extract Research Paper")
+
+    paper_type = st.selectbox(
+        "Select type of research paper:",
+        [
+            "Review Based Paper",
+            "Opinion/Perspective Based Paper",
+            "Empirical Research Paper",
+            "Research Paper (Other)",
+        ],
+    )
+
+    uploaded_file = st.file_uploader("Upload a PDF or text file", type=["pdf", "txt"])
+
+    if st.button("Extract & Save") and uploaded_file:
+        try:
+            # Read file content
+            if uploaded_file.type == "application/pdf":
+                pdf_reader = PyPDF2.PdfReader(uploaded_file)
+                text_content = ""
+                for page in pdf_reader.pages:
+                    text_content += page.extract_text()
+            else:
+                text_content = uploaded_file.read().decode("utf-8", errors="replace")
+
+            with st.spinner("Extracting fields..."):
+                data = process_paper(text_content, paper_type)
+
+            if data:
+                st.success(
+                    f"Paper extracted and saved to MongoDB in '{paper_type}' collection!"
+                )
+                st.write("Extracted fields:")
+                st.json(data)
+
+        except Exception as e:
+            st.error(f"An error occurred: {str(e)}")
+
+
+# ...existing code (if any)...
+
+if __name__ == "__main__":
+    main()

research3.py

@@ -0,0 +1,110 @@
+import streamlit as st
+import pandas as pd
+import requests
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+
+
+def call_perplexity_api(prompt: str) -> str:
+    """Call Perplexity AI with a prompt, return the text response if successful."""
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.3,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        st.error(f"API Error: {str(e)}")
+        return ""
+
+
+def generate_research_paper(df: pd.DataFrame, topic: str) -> dict:
+    """
+    For each column in the DataFrame, generate a research paper section (200-500 words)
+    that addresses the data in that column on the given topic. Return a dict: column -> text.
+    """
+    paper_sections = {}
+    for col in df.columns:
+        # Convert all non-null rows in the column to strings and join them for context
+        col_values = df[col].dropna().astype(str).tolist()
+        # We'll truncate if there's a ton of text
+        sample_text = " | ".join(col_values[:50])  # limit to first 50 rows for brevity
+
+        prompt = f"""
+        Topic: {topic}
+        Column: {col}
+        Data Samples: {sample_text}
+
+        Generate a well-structured research paper section that addresses the topic above,
+        referencing relevant information from the column data.
+        The section should be at least 100 words and at most 150 words.
+        Provide insights, examples, and possible research directions integrating the corpus data.
+        """
+        section_text = call_perplexity_api(prompt)
+        paper_sections[col] = section_text.strip() if section_text else ""
+    return paper_sections
+
+
+def format_paper(paper_dict: dict, topic: str) -> str:
+    """
+    Format the generated paper into a Markdown string.
+    Add the topic as the main title, each column name as a heading, and
+    the corresponding text as paragraph content.
+    """
+    md_text = f"# Research Paper on: {topic}\n\n"
+    for col, content in paper_dict.items():
+        md_text += f"## {col}\n{content}\n\n"
+    return md_text
+
+
+def main():
+    st.title("Topic + Corpus-Based Research Paper Generator")
+
+    topic_input = st.text_input("Enter the topic for the research paper:")
+    uploaded_file = st.file_uploader("Upload CSV corpus file", type="csv")
+
+    if uploaded_file:
+        df = pd.read_csv(uploaded_file)
+        st.write("### Preview of Uploaded Data")
+        st.dataframe(df.head())
+
+        if st.button("Generate Research Paper"):
+            if topic_input.strip():
+                st.info("Generating paper based on the topic and the corpus columns...")
+                with st.spinner("Calling Perplexity AI..."):
+                    paper = generate_research_paper(df, topic_input)
+                    if paper:
+                        formatted_paper = format_paper(paper, topic_input)
+                        st.success("Research Paper Generated Successfully!")
+                        st.write(formatted_paper)
+
+                        st.download_button(
+                            label="Download Paper as Markdown",
+                            data=formatted_paper,
+                            file_name="research_paper.md",
+                            mime="text/markdown",
+                        )
+                    else:
+                        st.error(
+                            "Paper generation failed. Please check Perplexity API key."
+                        )
+            else:
+                st.warning("Please enter a valid topic.")
+
+
+if __name__ == "__main__":
+    main()

research_assistant_dashboard.py

@@ -1,342 +1,349 @@
-import streamlit as st
-from openai import OpenAI
-import os
-from dotenv import load_dotenv
-from llama_index.core import (
-    VectorStoreIndex,
-    SimpleDirectoryReader,
-    Document,
-    GPTVectorStoreIndex,
-)
-from bson import ObjectId
-import requests
-import openai
-import numpy as np
-from pymongo import MongoClient
-from bson import ObjectId
-from datetime import datetime
-from llama_index.embeddings.openai import OpenAIEmbedding
-from typing import List, Dict
-
-# Initialize Perplexity API and OpenAI API
-load_dotenv()
-perplexity_api_key = os.getenv("PERPLEXITY_KEY")
-openai.api_key = os.getenv("OPENAI_KEY")
-
-# MongoDB setup
-MONGO_URI = os.getenv("MONGO_URI")
-client = MongoClient(MONGO_URI)
-db = client["novascholar_db"]
-research_papers_collection = db["research_papers"]
-
-
-def fetch_perplexity_data(api_key, topic):
-    """
-    Fetch research papers data from Perplexity API with proper formatting
-    """
-    headers = {
-        "accept": "application/json",
-        "content-type": "application/json",
-        "authorization": f"Bearer {api_key}",
-    }
-
-    # Structured prompt to get properly formatted response
-    messages = [
-        {
-            "role": "system",
-            "content": """You are a research paper retrieval expert. For the given topic, return exactly 10 research papers in the following format:
-            Title: Paper Title
-            Authors: Author 1, Author 2
-            Year: YYYY
-            Content: Detailed paper content with abstract and key findings
-            URL: DOI or paper URL
-            """,
-        },
-        {"role": "user", "content": f"Find 10 research papers about: {topic}"},
-    ]
-
-    try:
-        client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
-        response = client.chat.completions.create(
-            model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
-            messages=messages,
-        )
-
-        # Extract and validate response
-        content = response.choices[0].message.content
-        st.write("Fetched Data:", content)  # Debugging line to check the fetched data
-
-        return content
-
-    except Exception as e:
-        st.error(f"Failed to fetch data from Perplexity API: {str(e)}")
-        return ""
-
-
-def split_and_vectorize_papers(content: str) -> List[Dict]:
-    """Split and vectorize papers using OpenAI embeddings"""
-    papers = content.split("\n\n")
-
-    # Initialize OpenAI client
-    # client = OpenAI()  # Uses api_key from environment variable
-    vectors = []
-
-    for paper in papers:
-        try:
-            # Get embedding using OpenAI's API directly
-            response = openai.embeddings.create(
-                model="text-embedding-ada-002", input=paper, encoding_format="float"
-            )
-
-            # Extract embedding from response
-            embedding = response.data[0].embedding
-
-            vectors.append(
-                {"content": paper, "vector": embedding, "timestamp": datetime.utcnow()}
-            )
-
-        except Exception as e:
-            st.error(f"Error vectorizing paper: {str(e)}")
-            continue
-
-    return vectors
-
-
-def store_papers_in_mongodb(papers):
-    """Store papers with vectors in MongoDB"""
-    try:
-        for paper in papers:
-            # Prepare MongoDB document
-            mongo_doc = {
-                "content": paper["content"],
-                "vector": paper["vector"],
-                "created_at": datetime.utcnow(),
-            }
-
-            # Insert into MongoDB
-            db.papers.update_one(
-                {"content": paper["content"]}, {"$set": mongo_doc}, upsert=True
-            )
-
-        st.success(f"Stored {len(papers)} papers in database")
-        return True
-    except Exception as e:
-        st.error(f"Error storing papers: {str(e)}")
-
-
-def get_research_papers(query):
-    """
-    Get and store research papers with improved error handling
-    """
-    # Fetch papers from Perplexity
-    content = fetch_perplexity_data(perplexity_api_key, query)
-
-    if not content:
-        return []
-
-    # Split and vectorize papers
-    papers = split_and_vectorize_papers(content)
-
-    # Store papers in MongoDB
-    if store_papers_in_mongodb(papers):
-        return papers
-    else:
-        st.warning("Failed to store papers in database, but returning fetched results")
-        return papers
-
-
-def analyze_research_gaps(papers):
-    """
-    Analyze research gaps with improved prompt and error handling
-    """
-    if not papers:
-        return "No papers provided for analysis"
-
-    # Prepare paper summaries for analysis
-    paper_summaries = "\n\n".join(
-        [
-            f"Key Findings: {paper['content'][:500]}..."
-            # f"Title: {paper['title']}\nYear: {paper['year']}\nKey Findings: {paper['content'][:500]}..."
-            for paper in papers
-        ]
-    )
-
-    headers = {
-        "Authorization": f"Bearer {perplexity_api_key}",
-        "Content-Type": "application/json",
-    }
-
-    data = {
-        "messages": [
-            {
-                "role": "system",
-                "content": "You are a research analysis expert. Identify specific research gaps and future research directions based on the provided papers. Format your response with clear sections: Current State, Identified Gaps, and Future Directions.",
-            },
-            {
-                "role": "user",
-                "content": f"Analyze these papers and identify research gaps:\n\n{paper_summaries}",
-            },
-        ]
-    }
-
-    try:
-        client = OpenAI(
-            api_key=perplexity_api_key, base_url="https://api.perplexity.ai"
-        )
-        response = client.chat.completions.create(
-            model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
-            messages=data["messages"],
-        )
-        return response.choices[0].message.content
-
-    except Exception as e:
-        st.error(f"Failed to analyze research gaps: {str(e)}")
-        return "Error analyzing research gaps"
-
-
-def create_research_paper(gaps, topic, papers):
-    """
-    Create a research paper that addresses the identified gaps using Perplexity API
-    """
-    full_texts = "\n\n".join([paper["content"] for paper in papers])
-    headers = {
-        "Authorization": f"Bearer {perplexity_api_key}",
-        "Content-Type": "application/json",
-    }
-    data = {
-        "messages": [
-            {
-                "role": "system",
-                "content": "You are a research paper generation expert. Create a comprehensive research paper that addresses the identified gaps based on the provided papers. Format your response with clear sections: Introduction, Literature Review, Methodology, Results, Discussion, Conclusion, and References.",
-            },
-            {
-                "role": "user",
-                "content": f"Create a research paper on the topic '{topic}' that addresses the following research gaps:\n\n{gaps}\n\nBased on the following papers:\n\n{full_texts}",
-            },
-        ]
-    }
-    try:
-        client = OpenAI(
-            api_key=perplexity_api_key, base_url="https://api.perplexity.ai"
-        )
-        response = client.chat.completions.create(
-            model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
-            messages=data["messages"],
-        )
-        return response.choices[0].message.content
-
-    except Exception as e:
-        st.error(f"Failed to create research paper: {str(e)}")
-        return "Error creating research paper"
-
-
-def cosine_similarity(vec1, vec2):
-    """Calculate the cosine similarity between two vectors"""
-    vec1 = np.array(vec1)
-    vec2 = np.array(vec2)
-    return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
-
-
-def calculate_cosine_similarity(vec1: List[float], vec2: List[float]) -> float:
-    """Calculate cosine similarity between two vectors"""
-    return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
-
-
-def display_research_assistant_dashboard():
-    """Display research assistant dashboard"""
-    # Initialize session state for recommendations
-    if "recommendations" not in st.session_state:
-        st.session_state.recommendations = None
-    if "vectors" not in st.session_state:
-        st.session_state.vectors = None
-    if "generated_paper" not in st.session_state:
-        st.session_state.generated_paper = None
-
-    # Sidebar
-    with st.sidebar:
-        st.title(f"Welcome, {st.session_state.username}")
-        if st.button("Logout", use_container_width=True):
-            for key in st.session_state.keys():
-                del st.session_state[key]
-            st.rerun()
-
-    # Main content
-    st.title("Research Paper Recommendations")
-    search_query = st.text_input("Enter research topic:")
-    col1, col2 = st.columns(2)
-    with col1:
-        if st.button("Get Research Papers"):
-            if search_query:
-                with st.spinner("Fetching recommendations..."):
-                    st.session_state.recommendations = get_research_papers(search_query)
-                    st.session_state.vectors = [
-                        paper["vector"] for paper in st.session_state.recommendations
-                    ]
-                    st.markdown(
-                        "\n\n".join(
-                            [
-                                f"**{i+1}.**\n{paper['content']}"
-                                # f"**{i+1}. {paper['title']}**\n{paper['content']}"
-                                for i, paper in enumerate(
-                                    st.session_state.recommendations
-                                )
-                            ]
-                        )
-                    )
-            else:
-                st.warning("Please enter a search query")
-    with col2:
-        if st.button("Analyze Research Gaps"):
-            if st.session_state.recommendations:
-                with st.spinner("Analyzing research gaps..."):
-                    gaps = analyze_research_gaps(st.session_state.recommendations)
-                    st.session_state.generated_paper = create_research_paper(
-                        gaps, search_query, st.session_state.recommendations
-                    )
-                    st.markdown("### Potential Research Gaps")
-                    st.markdown(gaps)
-            else:
-                st.warning("Please get research papers first")
-
-    if st.button("Save and Vectorize"):
-        if st.session_state.generated_paper:
-            try:
-                # Initialize OpenAI client
-
-                # Get embedding for generated paper
-                response = openai.embeddings.create(
-                    model="text-embedding-ada-002",
-                    input=st.session_state.generated_paper,
-                    encoding_format="float",
-                )
-                generated_vector = response.data[0].embedding
-
-                # Calculate similarities with stored vectors
-                similarities = [
-                    calculate_cosine_similarity(generated_vector, paper_vector)
-                    for paper_vector in st.session_state.vectors
-                ]
-
-                # Display results
-                st.markdown("### Generated Research Paper")
-                st.markdown(st.session_state.generated_paper)
-
-                st.markdown("### Cosine Similarities with Original Papers")
-                for i, similarity in enumerate(similarities):
-                    st.metric(
-                        f"Paper {i+1}",
-                        value=f"{similarity:.3f}",
-                        help="Cosine similarity (1.0 = identical, 0.0 = completely different)",
-                    )
-
-            except Exception as e:
-                st.error(f"Error during vectorization: {str(e)}")
-        else:
-            st.warning("Please analyze research gaps first")
-
-
-# Run the dashboard
-if __name__ == "__main__":
-    display_research_assistant_dashboard()
-
-
+# import streamlit as st
+# from openai import OpenAI
+# import os
+# from dotenv import load_dotenv
+# from llama_index.core import (
+#     VectorStoreIndex,
+#     SimpleDirectoryReader,
+#     Document,
+#     GPTVectorStoreIndex,
+# )
+# from bson import ObjectId
+# import requests
+# import openai
+# import numpy as np
+# from pymongo import MongoClient
+# from bson import ObjectId
+# from datetime import datetime
+# from llama_index.embeddings.openai import OpenAIEmbedding
+# from typing import List, Dict
+
+# # Initialize Perplexity API and OpenAI API
+# load_dotenv()
+# perplexity_api_key = os.getenv("PERPLEXITY_KEY")
+# openai.api_key = os.getenv("OPENAI_KEY")
+
+# # MongoDB setup
+# MONGO_URI = os.getenv("MONGO_URI")
+# client = MongoClient(MONGO_URI)
+# db = client["novascholar_db"]
+# research_papers_collection = db["research_papers"]
+
+
+# def fetch_perplexity_data(api_key, topic):
+#     """
+#     Fetch research papers data from Perplexity API with proper formatting
+#     """
+#     headers = {
+#         "accept": "application/json",
+#         "content-type": "application/json",
+#         "authorization": f"Bearer {api_key}",
+#     }
+
+#     # Structured prompt to get properly formatted response
+#     messages = [
+#         {
+#             "role": "system",
+#             "content": """You are a research paper retrieval expert. For the given topic, return exactly 10 research papers in the following format:
+#             Title: Paper Title
+#             Authors: Author 1, Author 2
+#             Year: YYYY
+#             Content: Detailed paper content with abstract and key findings
+#             URL: DOI or paper URL
+#             """,
+#         },
+#         {"role": "user", "content": f"Find 10 research papers about: {topic}"},
+#     ]
+
+#     try:
+#         client = OpenAI(api_key=api_key, base_url="https://api.perplexity.ai")
+#         response = client.chat.completions.create(
+#             model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
+#             messages=messages,
+#         )
+
+#         # Extract and validate response
+#         content = response.choices[0].message.content
+#         st.write("Fetched Data:", content)  # Debugging line to check the fetched data
+
+#         return content
+
+#     except Exception as e:
+#         st.error(f"Failed to fetch data from Perplexity API: {str(e)}")
+#         return ""
+
+
+# def split_and_vectorize_papers(content: str) -> List[Dict]:
+#     """Split and vectorize papers using OpenAI embeddings"""
+#     papers = content.split("\n\n")
+
+#     # Initialize OpenAI client
+#     # client = OpenAI()  # Uses api_key from environment variable
+#     vectors = []
+
+#     for paper in papers:
+#         try:
+#             # Get embedding using OpenAI's API directly
+#             response = openai.embeddings.create(
+#                 model="text-embedding-ada-002", input=paper, encoding_format="float"
+#             )
+
+#             # Extract embedding from response
+#             embedding = response.data[0].embedding
+
+#             vectors.append(
+#                 {"content": paper, "vector": embedding, "timestamp": datetime.utcnow()}
+#             )
+
+#         except Exception as e:
+#             st.error(f"Error vectorizing paper: {str(e)}")
+#             continue
+
+#     return vectors
+
+
+# def store_papers_in_mongodb(papers):
+#     """Store papers with vectors in MongoDB"""
+#     try:
+#         for paper in papers:
+#             # Prepare MongoDB document
+#             mongo_doc = {
+#                 "content": paper["content"],
+#                 "vector": paper["vector"],
+#                 "created_at": datetime.utcnow(),
+#             }
+
+#             # Insert into MongoDB
+#             db.papers.update_one(
+#                 {"content": paper["content"]}, {"$set": mongo_doc}, upsert=True
+#             )
+
+#         st.success(f"Stored {len(papers)} papers in database")
+#         return True
+#     except Exception as e:
+#         st.error(f"Error storing papers: {str(e)}")
+
+
+# def get_research_papers(query):
+#     """
+#     Get and store research papers with improved error handling
+#     """
+#     # Fetch papers from Perplexity
+#     content = fetch_perplexity_data(perplexity_api_key, query)
+
+#     if not content:
+#         return []
+
+#     # Split and vectorize papers
+#     papers = split_and_vectorize_papers(content)
+
+#     # Store papers in MongoDB
+#     if store_papers_in_mongodb(papers):
+#         return papers
+#     else:
+#         st.warning("Failed to store papers in database, but returning fetched results")
+#         return papers
+
+
+# def analyze_research_gaps(papers):
+#     """
+#     Analyze research gaps with improved prompt and error handling
+#     """
+#     if not papers:
+#         return "No papers provided for analysis"
+
+#     # Prepare paper summaries for analysis
+#     paper_summaries = "\n\n".join(
+#         [
+#             f"Key Findings: {paper['content'][:500]}..."
+#             # f"Title: {paper['title']}\nYear: {paper['year']}\nKey Findings: {paper['content'][:500]}..."
+#             for paper in papers
+#         ]
+#     )
+
+#     headers = {
+#         "Authorization": f"Bearer {perplexity_api_key}",
+#         "Content-Type": "application/json",
+#     }
+
+#     data = {
+#         "messages": [
+#             {
+#                 "role": "system",
+#                 "content": "You are a research analysis expert. Identify specific research gaps and future research directions based on the provided papers. Format your response with clear sections: Current State, Identified Gaps, and Future Directions.",
+#             },
+#             {
+#                 "role": "user",
+#                 "content": f"Analyze these papers and identify research gaps:\n\n{paper_summaries}",
+#             },
+#         ]
+#     }
+
+#     try:
+#         client = OpenAI(
+#             api_key=perplexity_api_key, base_url="https://api.perplexity.ai"
+#         )
+#         response = client.chat.completions.create(
+#             model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
+#             messages=data["messages"],
+#         )
+#         return response.choices[0].message.content
+
+#     except Exception as e:
+#         st.error(f"Failed to analyze research gaps: {str(e)}")
+#         return "Error analyzing research gaps"
+
+
+# def create_research_paper(gaps, topic, papers):
+#     """
+#     Create a research paper that addresses the identified gaps using Perplexity API
+#     """
+#     full_texts = "\n\n".join([paper["content"] for paper in papers])
+#     headers = {
+#         "Authorization": f"Bearer {perplexity_api_key}",
+#         "Content-Type": "application/json",
+#     }
+#     data = {
+#         "messages": [
+#             {
+#                 "role": "system",
+#                 "content": "You are a research paper generation expert. Create a comprehensive research paper that addresses the identified gaps based on the provided papers. Format your response with clear sections: Introduction, Literature Review, Methodology, Results, Discussion, Conclusion, and References.",
+#             },
+#             {
+#                 "role": "user",
+#                 "content": f"Create a research paper on the topic '{topic}' that addresses the following research gaps:\n\n{gaps}\n\nBased on the following papers:\n\n{full_texts}",
+#             },
+#         ]
+#     }
+#     try:
+#         client = OpenAI(
+#             api_key=perplexity_api_key, base_url="https://api.perplexity.ai"
+#         )
+#         response = client.chat.completions.create(
+#             model="llama-3.1-sonar-small-128k-chat",  # Use the best Perplexity model
+#             messages=data["messages"],
+#         )
+#         return response.choices[0].message.content
+
+#     except Exception as e:
+#         st.error(f"Failed to create research paper: {str(e)}")
+#         return "Error creating research paper"
+
+
+# def cosine_similarity(vec1, vec2):
+#     """Calculate the cosine similarity between two vectors"""
+#     vec1 = np.array(vec1)
+#     vec2 = np.array(vec2)
+#     return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+
+# def calculate_cosine_similarity(vec1: List[float], vec2: List[float]) -> float:
+#     """Calculate cosine similarity between two vectors"""
+#     return np.dot(vec1, vec2) / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+
+# def display_research_assistant_dashboard():
+#     """Display research assistant dashboard"""
+#     # Initialize session state for recommendations
+#     if "recommendations" not in st.session_state:
+#         st.session_state.recommendations = None
+#     if "vectors" not in st.session_state:
+#         st.session_state.vectors = None
+#     if "generated_paper" not in st.session_state:
+#         st.session_state.generated_paper = None
+
+#     # Sidebar
+#     with st.sidebar:
+#         st.title(f"Welcome, {st.session_state.username}")
+#         if st.button("Logout", use_container_width=True):
+#             for key in st.session_state.keys():
+#                 del st.session_state[key]
+#             st.rerun()
+
+#     # Main content
+#     st.title("Research Paper Recommendations")
+#     search_query = st.text_input("Enter research topic:")
+#     col1, col2 = st.columns(2)
+#     with col1:
+#         if st.button("Get Research Papers"):
+#             if search_query:
+#                 with st.spinner("Fetching recommendations..."):
+#                     st.session_state.recommendations = get_research_papers(search_query)
+#                     st.session_state.vectors = [
+#                         paper["vector"] for paper in st.session_state.recommendations
+#                     ]
+#                     st.markdown(
+#                         "\n\n".join(
+#                             [
+#                                 f"**{i+1}.**\n{paper['content']}"
+#                                 # f"**{i+1}. {paper['title']}**\n{paper['content']}"
+#                                 for i, paper in enumerate(
+#                                     st.session_state.recommendations
+#                                 )
+#                             ]
+#                         )
+#                     )
+#             else:
+#                 st.warning("Please enter a search query")
+#     with col2:
+#         if st.button("Analyze Research Gaps"):
+#             if st.session_state.recommendations:
+#                 with st.spinner("Analyzing research gaps..."):
+#                     gaps = analyze_research_gaps(st.session_state.recommendations)
+#                     st.session_state.generated_paper = create_research_paper(
+#                         gaps, search_query, st.session_state.recommendations
+#                     )
+#                     st.markdown("### Potential Research Gaps")
+#                     st.markdown(gaps)
+#             else:
+#                 st.warning("Please get research papers first")
+
+#     if st.button("Save and Vectorize"):
+#         if st.session_state.generated_paper:
+#             try:
+#                 # Initialize OpenAI client
+
+#                 # Get embedding for generated paper
+#                 response = openai.embeddings.create(
+#                     model="text-embedding-ada-002",
+#                     input=st.session_state.generated_paper,
+#                     encoding_format="float",
+#                 )
+#                 generated_vector = response.data[0].embedding
+
+#                 # Calculate similarities with stored vectors
+#                 similarities = [
+#                     calculate_cosine_similarity(generated_vector, paper_vector)
+#                     for paper_vector in st.session_state.vectors
+#                 ]
+
+#                 # Display results
+#                 st.markdown("### Generated Research Paper")
+#                 st.markdown(st.session_state.generated_paper)
+
+#                 st.markdown("### Cosine Similarities with Original Papers")
+#                 for i, similarity in enumerate(similarities):
+#                     st.metric(
+#                         f"Paper {i+1}",
+#                         value=f"{similarity:.3f}",
+#                         help="Cosine similarity (1.0 = identical, 0.0 = completely different)",
+#                     )
+
+#             except Exception as e:
+#                 st.error(f"Error during vectorization: {str(e)}")
+#         else:
+#             st.warning("Please analyze research gaps first")
+
+
+# # Run the dashboard
+# if __name__ == "__main__":
+#     display_research_assistant_dashboard()
+
+import research_combine2
+# if __name__ == "__main__":
+#     display_research_assistant_dashboard()
+def display_research_assistant_dashboard():
+    research_combine2.display_research_assistant_dashboard()
+    
+    
+    

research_combine.py

@@ -0,0 +1,188 @@
+import streamlit as st
+import os
+import json
+import requests
+from dotenv import load_dotenv
+from pymongo import MongoClient
+from typing import Dict, Any
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+
+# MongoDB setup
+client = MongoClient(MONGODB_URI)
+db = client["novascholar_db"]
+collection = db["research_papers"]
+
+
+def search_papers(topic: str, num_papers: int) -> str:
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    prompt = f"""Find {num_papers} recent research papers about {topic}.
+    Return ONLY a valid JSON array with the following structure for each paper, no additional text:
+    [
+        {{
+            "Title": "paper title",
+            "Publication": "publication name",
+            "Journal_Conference": "venue name",
+            "Abstract": "abstract text",
+            "Keywords": "key terms",
+            "Author": "author names",
+            "Date_of_Publication": "publication date",
+            "Intro": "introduction summary",
+            "Literature_Review": "literature review summary",
+            "Research_Models_Used": "models description", 
+            "Methodology": "methodology description",
+            "Discussion": "discussion summary",
+            "Future_Scope": "future work",
+            "Theory": "theoretical framework",
+            "Independent_Variables": "list of variables",
+            "nof_Independent_Variables": 0,
+            "Dependent_Variables": "list of variables",
+            "nof_Dependent_Variables": 0,
+            "Control_Variables": "list of variables",
+            "nof_Control_Variables": 0,
+            "Extraneous_Variables": "list of variables",
+            "nof_Extraneous_Variables": 0
+        }}
+    ]"""
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are a research paper analyzer that returns only valid JSON arrays.",
+            },
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.1,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+
+        # Clean response and ensure it's valid JSON
+        content = content.strip()
+        if not content.startswith("["):
+            content = content[content.find("[") :]
+        if not content.endswith("]"):
+            content = content[: content.rfind("]") + 1]
+
+        # Validate JSON
+        papers = json.loads(content)
+        if not isinstance(papers, list):
+            raise ValueError("Response is not a JSON array")
+
+        # Insert into MongoDB
+        if papers:
+            collection.insert_many(papers)
+            return content
+        return "[]"
+
+    except json.JSONDecodeError as e:
+        st.error(f"Invalid JSON response: {str(e)}")
+        return None
+    except Exception as e:
+        st.error(f"Error: {str(e)}")
+        return None
+
+
+import research22
+import keywords_database_download
+import new_keywords
+import infranew
+import loldude
+import new_research_paper
+import research3
+import entire_download
+
+
+def main():
+    st.set_page_config(page_title="Research Papers", layout="wide")
+
+    st.title("Research Papers")
+
+    # Sidebar radio
+    option = st.sidebar.radio(
+        "Select an option",
+        [
+            "Search Papers",
+            "Upload Paper",
+            "Single Keyword Search",
+            "Multiple Keywords Search",
+            "Knowledge Graph",
+            "Cosine Similarity",
+            "Paper Generator",
+            "Paper from Topic",
+            "Download Entire Corpus",
+        ],
+    )
+
+    if option == "Search Papers":
+        st.subheader("Search and Store Papers")
+
+        topic = st.text_input("Enter research topic")
+        num_papers = st.number_input(
+            "Number of papers", min_value=1, max_value=10, value=5
+        )
+
+        if st.button("Search and Store"):
+            if topic:
+                with st.spinner(f"Searching and storing papers about {topic}..."):
+                    results = search_papers(topic, num_papers)
+                    if results:
+                        st.success(
+                            f"Successfully stored {num_papers} papers in MongoDB"
+                        )
+                        # Display results
+                        papers = json.loads(results)
+                        for paper in papers:
+                            with st.expander(paper["Title"]):
+                                for key, value in paper.items():
+                                    if key != "Title":
+                                        st.write(f"**{key}:** {value}")
+            else:
+                st.warning("Please enter a research topic")
+
+        # Add MongoDB connection status
+        if st.sidebar.button("Check Database Connection"):
+            try:
+                client.admin.command("ping")
+                print(MONGODB_URI)
+                st.sidebar.success("Connected to MongoDB")
+            except Exception as e:
+                st.sidebar.error(f"MongoDB Connection Error: {str(e)}")
+    elif option == "Single Keyword Search":
+        keywords_database_download.main()
+    elif option == "Multiple Keywords Search":
+        new_keywords.main()
+    elif option == "Knowledge Graph":
+        infranew.main()
+    elif option == "Cosine Similarity":
+        loldude.main()
+    elif option == "Paper Generator":
+        new_research_paper.main()
+    elif option == "Paper from Topic":
+        research3.main()
+    elif option == "Download Entire Corpus":
+        entire_download.main()
+    else:
+        # st.subheader("Blank Page")
+        # st.write("This is a placeholder for alternative content.")
+        research22.main()
+
+
+if __name__ == "__main__":
+    main()

research_combine2.py

@@ -0,0 +1,269 @@
+import new_research_paper
+import research3
+import entire_download
+import streamlit as st
+import os
+import json
+import requests
+from dotenv import load_dotenv
+from pymongo import MongoClient
+from typing import Dict, Any
+import research22
+import keywords_database_download
+import new_keywords
+import infranew
+import loldude
+import new_research_paper
+import research3
+import entire_download
+import sciclone
+import extract
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+MONGODB_URI = os.getenv(
+    "MONGODB_UR",
+    "mongodb+srv://milind:[email protected]/?retryWrites=true&w=majority&appName=Cluster0",
+)
+
+# MongoDB setup
+client = MongoClient(MONGODB_URI)
+db = client["novascholar_db"]
+
+
+def search_papers(topic: str, num_papers: int, paper_type: str) -> str:
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    attributes = {
+        "Review Based Paper": [
+            "Title",
+            "Publication",
+            "Journal_Conference",
+            "Abstract",
+            "Keywords",
+            "Author",
+            "Date_of_Publication",
+            "Intro",
+            "Literature_Review",
+            "Body",
+            "Protocol",
+            "Search String",
+            "Included Studies",
+            "Data Collection and Analysis Methods",
+            "Data Extraction Table",
+            "Synthesis and Analysis",
+            "Conclusion",
+            "Limitations",
+            "Results",
+            "References",
+            "Risk of Bias Assessment",
+        ],
+        "Opinion/Perspective Based Paper": [
+            "Title",
+            "Publication",
+            "Journal_Conference",
+            "Abstract",
+            "Keywords",
+            "Author",
+            "Date_of_Publication",
+            "Intro",
+            "Literature_Review",
+            "Introduction",
+            "Body",
+            "Results and Discussion",
+            "Conclusion",
+            "References",
+        ],
+        "Empirical Research Paper": [
+            "Title",
+            "Publication",
+            "Journal_Conference",
+            "Abstract",
+            "Keywords",
+            "Author",
+            "Date_of_Publication",
+            "Intro",
+            "Literature_Review",
+            "Introduction",
+            "Body",
+            "Methodology",
+            "Participants",
+            "Survey Instrument",
+            "Data Collection",
+            "Data Analysis",
+            "Results and Discussion",
+            "Conclusion",
+            "References",
+        ],
+        "Research Paper (Other)": [
+            "Title",
+            "Publication",
+            "Journal_Conference",
+            "Abstract",
+            "Keywords",
+            "Author",
+            "Date_of_Publication",
+            "Intro",
+            "Literature_Review",
+            "Research_Models_Used",
+            "Methodology",
+            "Discussion",
+            "Future_Scope",
+            "Theory",
+            "Independent_Variables",
+            "nof_Independent_Variables",
+            "Dependent_Variables",
+            "nof_Dependent_Variables",
+            "Control_Variables",
+            "Extraneous_Variables",
+            "nof_Control_Variables",
+            "nof_Extraneous_Variables",
+        ],
+    }
+
+    selected_attributes = attributes[paper_type]
+    prompt = f"""Find {num_papers} recent research papers about {topic}.
+    Return ONLY a valid JSON array with the following structure for each paper, no additional text:
+    [{{
+        {", ".join([f'"{attr}": "value"' for attr in selected_attributes])}
+    }}]"""
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are a research paper analyzer that returns only valid JSON arrays.",
+            },
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.1,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        content = response.json()["choices"][0]["message"]["content"]
+
+        # Clean response and ensure it's valid JSON
+        content = content.strip()
+        if not content.startswith("["):
+            content = content[content.find("[") :]
+        if not content.endswith("]"):
+            content = content[: content.rfind("]") + 1]
+
+        # Validate JSON
+        papers = json.loads(content)
+        if not isinstance(papers, list):
+            raise ValueError("Response is not a JSON array")
+
+        # Insert into MongoDB
+        collection = db[paper_type.replace(" ", "_").lower()]
+        if papers:
+            collection.insert_many(papers)
+            return content
+        return "[]"
+
+    except json.JSONDecodeError as e:
+        st.error(f"Invalid JSON response: {str(e)}")
+        return None
+    except Exception as e:
+        st.error(f"Error: {str(e)}")
+        return None
+
+
+def display_research_assistant_dashboard():
+    #st.set_page_config(page_title="Research Papers", layout="wide")
+
+   # st.title("Research Papers")
+
+    # Sidebar radio
+    option = st.sidebar.radio(
+        "Select an option",
+        [
+            "Search Papers",
+            "Upload Paper",
+            "Single Keyword Search",
+            "Multiple Keywords Search",
+            "Knowledge Graph",
+            "Cosine Similarity",
+            "Paper Generator",
+            "Paper from Topic",
+            "Download Entire Corpus",
+            "Research Copilot",
+            "Research Paper Analysis Tool",
+        ],
+    )
+
+    if option == "Search Papers":
+        st.subheader("Search and Store Papers")
+
+        topic = st.text_input("Enter research topic")
+        num_papers = st.number_input(
+            "Number of papers", min_value=1, max_value=10, value=5
+        )
+        paper_type = st.selectbox(
+            "Select type of research paper",
+            [
+                "Review Based Paper",
+                "Opinion/Perspective Based Paper",
+                "Empirical Research Paper",
+                "Research Paper (Other)",
+            ],
+        )
+
+        if st.button("Search and Store"):
+            if topic:
+                with st.spinner(f"Searching and storing papers about {topic}..."):
+                    results = search_papers(topic, num_papers, paper_type)
+                    if results:
+                        st.success(
+                            f"Successfully stored {num_papers} papers in MongoDB"
+                        )
+                        # Display results
+                        papers = json.loads(results)
+                        for paper in papers:
+                            with st.expander(paper["Title"]):
+                                for key, value in paper.items():
+                                    if key != "Title":
+                                        st.write(f"**{key}:** {value}")
+            else:
+                st.warning("Please enter a research topic")
+
+        # Add MongoDB connection status
+        if st.sidebar.button("Check Database Connection"):
+            try:
+                client.admin.command("ping")
+                print(MONGODB_URI)
+                st.sidebar.success("Connected to MongoDB")
+            except Exception as e:
+                st.sidebar.error(f"MongoDB Connection Error: {str(e)}")
+    elif option == "Single Keyword Search":
+        keywords_database_download.main()
+    elif option == "Multiple Keywords Search":
+        new_keywords.main()
+    elif option == "Knowledge Graph":
+        infranew.main()
+    elif option == "Cosine Similarity":
+        loldude.main()
+    elif option == "Paper Generator":
+        new_research_paper.main()
+    elif option == "Paper from Topic":
+        research3.main()
+    elif option == "Download Entire Corpus":
+        entire_download.main()
+    elif option == "Research Copilot":
+        sciclone.main()
+    elif option == "Research Paper Analysis Tool":
+        extract.main()
+    else:
+        research22.main()
+
+
+if __name__ == "__main__":
+    display_research_assistant_dashboard()

rubrics.py

@@ -0,0 +1,112 @@
+import streamlit as st
+from pymongo import MongoClient
+from openai import OpenAI
+from bson import ObjectId
+import json
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+MONGO_URI = os.getenv('MONGO_URI')
+OPENAI_API_KEY = os.getenv('OPENAI_KEY')
+
+client = MongoClient(MONGO_URI)
+db = client['novascholar_db']
+# db.create_collection("rubrics")
+rubrics_collection = db['rubrics']
+resources_collection = db['resources']
+courses_collection = db['courses']
+
+def generate_rubrics(api_key, session_title, outcome_description, taxonomy, pre_class_material):
+    prompt = f"""
+    You are an expert educational AI assistant specializing in instructional design. Generate a detailed rubric for the session titled "{session_title}". The rubric should be aligned with Bloom's Taxonomy level "{taxonomy}" and use numerical scoring levels (4,3,2,1) instead of descriptive levels. Use the following context:
+
+    Session Outcome Description:
+    {outcome_description}
+
+    Pre-class Material:
+    {pre_class_material}
+
+    Please generate the rubric in JSON format with these specifications:
+    1. Use numerical levels (4=Highest, 1=Lowest) instead of descriptive levels
+    2. Include 4-5 relevant criteria based on the session outcome
+    3. Each criterion should have clear descriptors for each numerical level
+    4. Focus on objectively measurable aspects for evaluation
+    5. Structure should be suitable for evaluating assignments and test answers
+
+    ***IMPORTANT: DO NOT INCLUDE THE WORD JSON IN THE OUTPUT STRING, DO NOT INCLUDE BACKTICKS (```) IN THE OUTPUT, AND DO NOT INCLUDE ANY OTHER TEXT, OTHER THAN THE ACTUAL JSON RESPONSE. START THE RESPONSE STRING WITH AN OPEN CURLY BRACE {{ AND END WITH A CLOSING CURLY BRACE }}.***
+    """
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are an expert educational AI assistant specializing in instructional design.",
+        },
+        {
+            "role": "user",
+            "content": prompt
+        },
+    ]
+
+    try:
+        client = OpenAI(api_key=api_key)
+        response = client.chat.completions.create(
+            model="gpt-4-0125-preview",
+            messages=messages
+        )
+        return response.choices[0].message.content
+    except Exception as e:
+        st.error(f"Failed to generate rubrics: {e}")
+        return None
+
+def display_rubrics_tab(session, course_id):
+    st.subheader("Generated Rubrics")
+
+    # Fetch session details from the courses collection
+    course_data = courses_collection.find_one(
+        {"course_id": course_id, "sessions.session_id": session['session_id']},
+        {"sessions.$": 1}
+    )
+    
+    if course_data and 'sessions' in course_data and len(course_data['sessions']) > 0:
+        session_data = course_data['sessions'][0]
+        
+        # Extract session learning outcomes
+        if 'session_learning_outcomes' in session_data and len(session_data['session_learning_outcomes']) > 0:
+            outcome = session_data['session_learning_outcomes'][0]
+            outcome_description = outcome.get('outcome_description', '')
+            taxonomy_level = outcome.get('bloom_taxonomy_level', '')
+            
+            # Display fetched information
+            st.markdown("### Session Information")
+            st.markdown(f"**Session Title:** {session['title']}")
+            st.markdown(f"**Learning Outcome:** {outcome_description}")
+            st.markdown(f"**Taxonomy Level:** {taxonomy_level}")
+
+            # Fetch pre-class material
+            pre_class_material_docs = resources_collection.find({"session_id": session['session_id']})
+            pre_class_material = "\n".join([f"{doc.get('title', 'No Title')}: {doc.get('url', 'No URL')}" for doc in pre_class_material_docs])
+
+            if st.button("Generate Rubric"):
+                rubric = generate_rubrics(
+                    OPENAI_API_KEY,
+                    session['title'],
+                    outcome_description,
+                    taxonomy_level,
+                    pre_class_material
+                )
+                
+                if rubric:
+                    st.json(rubric)
+                    if st.button("Save Rubric"):
+                        rubric_data = {
+                            "course_id": course_id,
+                            "session_id": session['session_id'],
+                            "rubric": json.loads(rubric)
+                        }
+                        rubrics_collection.insert_one(rubric_data)
+                        st.success("Rubric saved successfully!")
+        else:
+            st.error("No learning outcomes found for this session")
+    else:
+        st.error("Session data not found")

sciclone.py

@@ -0,0 +1,466 @@
+import streamlit as st
+import requests
+import PyPDF2
+from typing import Optional, Dict, List
+import json
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from concurrent.futures import ThreadPoolExecutor
+import xml.etree.ElementTree as ET
+import re
+from datetime import datetime
+import time
+from dotenv import load_dotenv
+import os
+import pandas as pd
+
+# Load environment variables
+load_dotenv()
+PERPLEXITY_API_KEY = os.getenv("PERPLEXITY_API_KEY")
+PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions"
+SAPLING_API_KEY = os.getenv("SAPLING_API_KEY")
+
+
+def call_perplexity_api(prompt: str) -> str:
+    """Call Perplexity AI with a prompt, return the text response if successful."""
+    headers = {
+        "Authorization": f"Bearer {PERPLEXITY_API_KEY}",
+        "Content-Type": "application/json",
+    }
+
+    payload = {
+        "model": "llama-3.1-sonar-small-128k-chat",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.3,
+    }
+
+    try:
+        response = requests.post(PERPLEXITY_API_URL, headers=headers, json=payload)
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
+    except Exception as e:
+        st.error(f"API Error: {str(e)}")
+        return ""
+
+
+def extract_text_from_pdf(pdf_file):
+    """Extract text content from a PDF file."""
+    pdf_reader = PyPDF2.PdfReader(pdf_file)
+    text = ""
+    for page in pdf_reader.pages:
+        text += page.extract_text() + "\n"
+    return text
+
+
+def analyze_paper(text: str, category: str) -> str:
+    """Generate a prompt and get analysis for a specific category."""
+    prompts = {
+        "Summarized Abstract": "Extract and summarize the abstract from this research paper:",
+        "Results": "What are the main results and findings from this research paper:",
+        "Summarized Introduction": "Summarize the introduction section of this research paper:",
+        "Methods Used": "What are the main methods and methodologies used in this research:",
+        "Literature Survey": "Summarize the literature review or related work from this paper:",
+        "Limitations": "What are the limitations mentioned in this research:",
+        "Contributions": "What are the main contributions of this research:",
+        "Practical Implications": "What are the practical implications of this research:",
+        "Objectives": "What are the main objectives of this research:",
+        "Findings": "What are the key findings from this research:",
+        "Future Research": "What future research directions are suggested in this paper:",
+        "Dependent Variables": "What are the dependent variables studied in this research:",
+        "Independent Variables": "What are the independent variables studied in this research:",
+        "Dataset": "What dataset(s) were used in this research:",
+        "Problem Statement": "What is the main problem statement or research question:",
+        "Challenges": "What challenges were faced or addressed in this research:",
+        "Applications": "What are the potential applications of this research:",
+    }
+
+    prompt = f"{prompts[category]}\n\nPaper text: {text[:5000]}"  # Limit text to avoid token limits
+    return call_perplexity_api(prompt)
+
+
+class ResearchAssistant:
+    def __init__(self, perplexity_key: str):
+        self.perplexity_key = perplexity_key
+
+    def chat_with_pdf(self, pdf_text: str, query: str) -> Dict:
+        chunks = self._split_text(pdf_text)
+        relevant_chunks = self._get_relevant_chunks(chunks, query)
+
+        prompt = f"Context from PDF:\n\n{relevant_chunks}\n\nQuestion: {query}"
+        response_text = call_perplexity_api(prompt)
+        return {"choices": [{"message": {"content": response_text}}]}
+
+    def generate_literature_review(self, topic: str) -> Dict:
+        try:
+            # Search arXiv for papers
+            papers = self._search_arxiv(topic)
+            if not papers:
+                return {"error": "No papers found on the topic"}
+
+            # Format paper information
+            papers_summary = "\n\n".join(
+                [
+                    f"Paper: {p['title']}\nAuthors: {', '.join(p['authors'])}\nSummary: {p['summary']}"
+                    for p in papers
+                ]
+            )
+
+            prompt = f"""Generate a comprehensive literature review on '{topic}'. Based on these papers:
+
+            {papers_summary}
+
+            Structure the review as follows:
+            1. Introduction and Background
+            2. Current Research Trends
+            3. Key Findings and Themes
+            4. Research Gaps
+            5. Future Directions"""
+
+            response_text = call_perplexity_api(prompt)
+            return {"choices": [{"message": {"content": response_text}}]}
+        except Exception as e:
+            return {"error": f"Literature review generation failed: {str(e)}"}
+
+    def ai_writer(self, outline: str, references: List[str]) -> Dict:
+        prompt = f"""Write a research paper following this structure:
+        
+        Outline:
+        {outline}
+        
+        References to incorporate:
+        {json.dumps(references)}
+        
+        Instructions:
+        - Follow academic writing style
+        - Include appropriate citations
+        - Maintain logical flow
+        - Include introduction and conclusion"""
+
+        response_text = call_perplexity_api(prompt)
+        return {"choices": [{"message": {"content": response_text}}]}
+
+    def refine_response(self, response: str, column: str) -> str:
+        prompt = f"""Refine the following response to fit the '{column}' column in a research paper CSV format:
+        
+        Response: {response}
+        
+        Ensure the response is clear, concise, and fits the context of the column."""
+
+        refined_response = call_perplexity_api(prompt)
+        return refined_response
+
+    def paraphrase(self, text: str) -> Dict:
+        prompt = f"""Paraphrase the following text while:
+        - Maintaining academic tone
+        - Preserving key meaning
+        - Improving clarity
+        
+        Text: {text}"""
+
+        response_text = call_perplexity_api(prompt)
+        return {"choices": [{"message": {"content": response_text}}]}
+
+    def generate_citation(self, paper_info: Dict, style: str = "APA") -> Dict:
+        prompt = f"""Generate a {style} citation for:
+        Title: {paper_info['title']}
+        Authors: {', '.join(paper_info['authors'])}
+        Year: {paper_info['year']}
+        
+        Follow exact {style} format guidelines."""
+
+        response_text = call_perplexity_api(prompt)
+        return {"citation": response_text}
+
+    def detect_ai_content(self, text: str) -> Dict:
+        prompt = f"""You are an AI content detector. Analyze the text for:
+        1. Writing style consistency
+        2. Language patterns
+        3. Contextual coherence
+        4. Common AI patterns
+        Provide a clear analysis with confidence level.
+        
+        Text: {text}"""
+
+        response = requests.post(
+            "https://api.sapling.ai/api/v1/aidetect",
+            json={"key": SAPLING_API_KEY, "text": text},
+        )
+        st.info(
+            "A score from 0 to 1 will be returned, with 0 indicating the maximum confidence that the text is human-written, and 1 indicating the maximum confidence that the text is AI-generated."
+        )
+
+        if response.status_code == 200:
+            return {"choices": [{"message": {"content": response.json()}}]}
+        else:
+            return {
+                "error": f"Sapling API Error: {response.status_code} - {response.text}"
+            }
+
+    def _split_text(self, text: str) -> List[str]:
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", ". ", " ", ""]
+        )
+        return splitter.split_text(text)
+
+    def _get_relevant_chunks(self, chunks: List[str], query: str) -> str:
+        # Simple keyword-based relevance scoring
+        query_words = set(query.lower().split())
+        scored_chunks = []
+
+        for chunk in chunks:
+            chunk_words = set(chunk.lower().split())
+            score = len(query_words.intersection(chunk_words))
+            scored_chunks.append((score, chunk))
+
+        scored_chunks.sort(reverse=True)
+        return "\n\n".join(chunk for _, chunk in scored_chunks[:3])
+
+    def _search_arxiv(self, topic: str) -> List[Dict]:
+        try:
+            query = "+AND+".join(topic.split())
+            url = f"http://export.arxiv.org/api/query?search_query=all:{query}&start=0&max_results=5"
+            response = requests.get(url, timeout=10)
+            response.raise_for_status()
+            return self._parse_arxiv_response(response.text)
+        except Exception as e:
+            print(f"arXiv search failed: {str(e)}")
+            return []
+
+    def _parse_arxiv_response(self, response_text: str) -> List[Dict]:
+        try:
+            root = ET.fromstring(response_text)
+            papers = []
+            for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
+                paper = {
+                    "id": entry.find("{http://www.w3.org/2005/Atom}id").text,
+                    "title": entry.find(
+                        "{http://www.w3.org/2005/Atom}title"
+                    ).text.strip(),
+                    "summary": entry.find(
+                        "{http://www.w3.org/2005/Atom}summary"
+                    ).text.strip(),
+                    "authors": [
+                        author.find("{http://www.w3.org/2005/Atom}name").text.strip()
+                        for author in entry.findall(
+                            "{http://www.w3.org/2005/Atom}author"
+                        )
+                    ],
+                    "published": entry.find(
+                        "{http://www.w3.org/2005/Atom}published"
+                    ).text[:10],
+                }
+                papers.append(paper)
+            return papers
+        except Exception as e:
+            print(f"arXiv response parsing failed: {str(e)}")
+            return []
+
+
+def main():
+    # st.set_page_config(page_title="Research Assistant", layout="wide")
+    st.title("Research Copilot")
+
+    if not PERPLEXITY_API_KEY:
+        st.warning("Perplexity API key not found in environment variables.")
+        return
+
+    assistant = ResearchAssistant(PERPLEXITY_API_KEY)
+
+    tabs = st.tabs(
+        [
+            "Chat with PDF",
+            "Literature Review",
+            "AI Writer",
+            "Extract Data",
+            "Paraphraser",
+            "Citation Generator",
+            "AI Detector",
+        ]
+    )
+
+    with tabs[0]:  # Chat with PDF
+        st.header("Chat with PDF")
+
+        # File uploader with clear button
+        col1, col2 = st.columns([3, 1])
+        with col1:
+            uploaded_file = st.file_uploader("Upload PDF", type="pdf", key="pdf_chat")
+        with col2:
+            if st.button("Clear PDF"):
+                st.session_state.pop("pdf_text", None)
+                st.rerun()
+
+        if uploaded_file:
+            if "pdf_text" not in st.session_state:
+                with st.spinner("Processing PDF..."):
+                    reader = PyPDF2.PdfReader(uploaded_file)
+                    st.session_state.pdf_text = ""
+                    for page in reader.pages:
+                        st.session_state.pdf_text += page.extract_text()
+                    st.success("PDF processed successfully!")
+
+            query = st.text_input("Ask a question about the PDF")
+            if query:
+                with st.spinner("Analyzing..."):
+                    response = assistant.chat_with_pdf(st.session_state.pdf_text, query)
+                    if "error" in response:
+                        st.error(response["error"])
+                    else:
+                        st.write(response["choices"][0]["message"]["content"])
+
+    with tabs[1]:  # Literature Review
+        st.header("Literature Review")
+        topic = st.text_input("Enter research topic")
+        if st.button("Generate Review") and topic:
+            with st.spinner("Generating literature review..."):
+                review = assistant.generate_literature_review(topic)
+                if "error" in review:
+                    st.error(review["error"])
+                else:
+                    st.write(review["choices"][0]["message"]["content"])
+
+    with tabs[2]:  # AI Writer
+        st.header("AI Writer")
+        outline = st.text_area("Enter paper outline")
+        references = st.text_area("Enter references (one per line)")
+        if st.button("Generate Paper") and outline:
+            with st.spinner("Writing paper..."):
+                paper = assistant.ai_writer(outline, references.split("\n"))
+                if "error" in paper:
+                    st.error(paper["error"])
+                else:
+                    st.write(paper["choices"][0]["message"]["content"])
+
+    with tabs[3]:  # Extract Data
+        st.header("Extract Data")
+
+        uploaded_files = st.file_uploader(
+            "Upload multiple PDF  files", type="pdf", accept_multiple_files=True
+        )
+
+        if uploaded_files:
+            if st.button("Process Papers"):
+                # Initialize progress bar
+                progress_bar = st.progress(0)
+                status_text = st.empty()
+
+                # Initialize results dictionary
+                results = []
+
+                # Define categories
+                categories = [
+                    "Summarized Abstract",
+                    "Results",
+                    "Summarized Introduction",
+                    "Methods Used",
+                    "Literature Survey",
+                    "Limitations",
+                    "Contributions",
+                    "Practical Implications",
+                    "Objectives",
+                    "Findings",
+                    "Future Research",
+                    "Dependent Variables",
+                    "Independent Variables",
+                    "Dataset",
+                    "Problem Statement",
+                    "Challenges",
+                    "Applications",
+                ]
+
+                # Process each file
+                for i, file in enumerate(uploaded_files):
+                    status_text.text(f"Processing {file.name}...")
+
+                    # Extract text from PDF
+                    text = extract_text_from_pdf(file)
+
+                    # Initialize paper results
+                    paper_results = {"Filename": file.name}
+
+                    # Analyze each category
+                    for j, category in enumerate(categories):
+                        status_text.text(f"Processing {file.name} - {category}")
+                        paper_results[category] = analyze_paper(text, category)
+
+                        # Update progress
+                        progress = (i * len(categories) + j + 1) / (
+                            len(uploaded_files) * len(categories)
+                        )
+                        progress_bar.progress(progress)
+
+                        # Add small delay to avoid API rate limits
+                        time.sleep(1)
+
+                    results.append(paper_results)
+
+                # Create DataFrame
+                df = pd.DataFrame(results)
+
+                # Convert DataFrame to CSV
+                csv = df.to_csv(index=False)
+
+                # Create download button
+                st.download_button(
+                    label="Download Results as CSV",
+                    data=csv,
+                    file_name="research_papers_analysis.csv",
+                    mime="text/csv",
+                )
+
+                # Display results in the app
+                st.subheader("Analysis Results")
+                st.dataframe(df)
+
+                status_text.text("Processing complete!")
+                progress_bar.progress(1.0)
+
+    with tabs[4]:  # Paraphraser
+        st.header("Paraphraser")
+        text = st.text_area("Enter text to paraphrase")
+        if st.button("Paraphrase") and text:
+            with st.spinner("Paraphrasing..."):
+                result = assistant.paraphrase(text)
+                if "error" in result:
+                    st.error(result["error"])
+                else:
+                    st.write(result["choices"][0]["message"]["content"])
+
+    with tabs[5]:  # Citation Generator
+        st.header("Citation Generator")
+        col1, col2 = st.columns(2)
+        with col1:
+            title = st.text_input("Paper Title")
+            authors = st.text_input("Authors (comma-separated)")
+        with col2:
+            year = st.text_input("Year")
+            style = st.selectbox("Citation Style", ["APA", "MLA", "Chicago"])
+
+        if st.button("Generate Citation") and title:
+            with st.spinner("Generating citation..."):
+                citation = assistant.generate_citation(
+                    {
+                        "title": title,
+                        "authors": [a.strip() for a in authors.split(",")],
+                        "year": year,
+                    },
+                    style,
+                )
+                if "error" in citation:
+                    st.error(citation["error"])
+                else:
+                    st.code(citation["citation"], language="text")
+
+    with tabs[6]:  # AI Detector
+        st.header("AI Detector")
+        text = st.text_area("Enter text to analyze")
+        if st.button("Detect AI Content") and text:
+            with st.spinner("Analyzing..."):
+                result = assistant.detect_ai_content(text)
+                if "error" in result:
+                    st.error(result["error"])
+                else:
+                    st.write(result["choices"][0]["message"]["content"])
+
+
+if __name__ == "__main__":
+    main()

subjective_test_evaluation.py

@@ -0,0 +1,247 @@
+import streamlit as st
+from datetime import datetime
+from pymongo import MongoClient
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+from bson import ObjectId
+
+load_dotenv()
+
+# MongoDB setup
+MONGO_URI = os.getenv('MONGO_URI')
+client = MongoClient(MONGO_URI)
+db = client["novascholar_db"]
+subjective_tests_collection = db["subjective_tests"]
+subjective_test_evaluation_collection = db["subjective_test_evaluation"]
+resources_collection = db["resources"]
+students_collection = db["students"]
+
+def evaluate_subjective_answers(session_id, student_id, test_id):
+    """
+    Generate evaluation and analysis for subjective test answers
+    """
+    try:
+        # Fetch test and student submission
+        test = subjective_tests_collection.find_one({"_id": test_id})
+        if not test:
+            return None
+
+        # Find student's submission
+        submission = next(
+            (sub for sub in test.get('submissions', []) 
+             if sub['student_id'] == str(student_id)),
+            None
+        )
+        if not submission:
+            return None
+
+        # Fetch pre-class materials
+        pre_class_materials = resources_collection.find({"session_id": session_id})
+        pre_class_content = ""
+        for material in pre_class_materials:
+            if 'text_content' in material:
+                pre_class_content += material['text_content'] + "\n"
+
+        # Default rubric (can be customized later)
+        default_rubric = """
+        1. Content Understanding (1-4):
+           - Demonstrates comprehensive understanding of core concepts
+           - Accurately applies relevant theories and principles
+           - Provides specific examples and evidence
+           
+        2. Critical Analysis (1-4):
+           - Shows depth of analysis
+           - Makes meaningful connections
+           - Demonstrates original thinking
+           
+        3. Organization & Clarity (1-4):
+           - Clear structure and flow
+           - Well-developed arguments
+           - Effective use of examples
+        """
+
+        # Initialize OpenAI client
+        client = OpenAI(api_key=os.getenv('OPENAI_KEY'))
+
+        evaluations = []
+        for i, (question, answer) in enumerate(zip(test['questions'], submission['answers'])):
+            analysis_content = f"""
+            Question: {question['question']}
+            Student Answer: {answer}
+            """
+
+            prompt_template = f"""As an educational assessor, evaluate this student's answer based on the provided rubric criteria and pre-class materials. Follow these assessment guidelines:
+            
+            1. Evaluation Process:
+            - Use each rubric criterion (scored 1-4) for internal assessment
+            - Compare response with pre-class materials
+            - Check alignment with all rubric requirements
+            - Calculate final score: sum of criteria scores converted to 10-point scale
+
+            Pre-class Materials:
+            {pre_class_content[:1000]}  # Truncate to avoid token limits
+
+            Rubric Criteria:
+            {default_rubric}
+
+            Question and Answer:
+            {analysis_content}
+
+            Provide your assessment in the following format:
+
+            **Score and Evidence**
+            - Score: [X]/10
+            - Evidence for deduction: [One-line reference to most significant gap or inaccuracy]
+
+            **Key Areas for Improvement**
+            - [Concise improvement point 1]
+            - [Concise improvement point 2]
+            - [Concise improvement point 3]
+            """
+
+            # Generate evaluation using OpenAI
+            response = client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[{"role": "user", "content": prompt_template}],
+                max_tokens=500,
+                temperature=0.4
+            )
+
+            evaluations.append({
+                "question_number": i + 1,
+                "question": question['question'],
+                "answer": answer,
+                "evaluation": response.choices[0].message.content
+            })
+
+        # Store evaluation in MongoDB
+        evaluation_doc = {
+            "test_id": test_id,
+            "student_id": student_id,
+            "session_id": session_id,
+            "evaluations": evaluations,
+            "evaluated_at": datetime.utcnow()
+        }
+        
+        subjective_test_evaluation_collection.insert_one(evaluation_doc)
+        return evaluation_doc
+
+    except Exception as e:
+        print(f"Error in evaluate_subjective_answers: {str(e)}")
+        return None
+
+def display_evaluation_to_faculty(session_id, student_id, course_id):
+    """
+    Display interface for faculty to generate and view evaluations
+    """
+    st.header("Evaluate Subjective Tests")
+
+    try:
+        # Fetch available tests
+        tests = list(subjective_tests_collection.find({
+            "session_id": str(session_id),
+            "status": "active"
+        }))
+
+        if not tests:
+            st.info("No subjective tests found for this session.")
+            return
+
+        # Select test
+        test_options = {
+            f"{test['title']} (Created: {test['created_at'].strftime('%Y-%m-%d %H:%M')})" if 'created_at' in test else test['title']: test['_id'] 
+            for test in tests
+        }
+        
+        if test_options:
+            selected_test = st.selectbox(
+                "Select Test to Evaluate",
+                options=list(test_options.keys())
+            )
+
+            if selected_test:
+                test_id = test_options[selected_test]
+                test = subjective_tests_collection.find_one({"_id": test_id})
+
+                if test:
+                    submissions = test.get('submissions', [])
+                    if not submissions:
+                        st.warning("No submissions found for this test.")
+                        return
+
+                    # Create a dropdown for student submissions
+                    student_options = {
+                        f"{students_collection.find_one({'_id': ObjectId(sub['student_id'])})['full_name']} (Submitted: {sub['submitted_at'].strftime('%Y-%m-%d %H:%M')})": sub['student_id']
+                        for sub in submissions
+                    }
+
+                    selected_student = st.selectbox(
+                        "Select Student Submission",
+                        options=list(student_options.keys())
+                    )
+
+                    if selected_student:
+                        student_id = student_options[selected_student]
+                        submission = next(sub for sub in submissions if sub['student_id'] == student_id)
+
+                        st.markdown(f"**Submission Date:** {submission.get('submitted_at', 'No submission date')}")
+                        st.markdown("---")
+
+                        # Display questions and answers
+                        st.subheader("Submission Details")
+                        for i, (question, answer) in enumerate(zip(test['questions'], submission['answers'])):
+                            st.markdown(f"**Question {i+1}:** {question['question']}")
+                            st.markdown(f"**Answer:** {answer}")
+                            st.markdown("---")
+
+                        # Check for existing evaluation
+                        existing_eval = subjective_test_evaluation_collection.find_one({
+                            "test_id": test_id,
+                            "student_id": student_id,
+                            "session_id": str(session_id)
+                        })
+
+                        if existing_eval:
+                            st.subheader("Evaluation Results")
+                            for eval_item in existing_eval['evaluations']:
+                                st.markdown(f"### Evaluation for Question {eval_item['question_number']}")
+                                st.markdown(eval_item['evaluation'])
+                                st.markdown("---")
+                            
+                            st.success("✓ Evaluation completed")
+                            if st.button("Regenerate Evaluation", key=f"regenerate_{student_id}_{test_id}"):
+                                with st.spinner("Regenerating evaluation..."):
+                                    evaluation = evaluate_subjective_answers(
+                                        str(session_id),
+                                        student_id,
+                                        test_id
+                                    )
+                                    if evaluation:
+                                        st.success("Evaluation regenerated successfully!")
+                                        st.rerun()
+                                    else:
+                                        st.error("Error regenerating evaluation.")
+                        else:
+                            st.subheader("Generate Evaluation")
+                            if st.button("Generate Evaluation", key=f"evaluate_{student_id}_{test_id}"):
+                                with st.spinner("Generating evaluation..."):
+                                    evaluation = evaluate_subjective_answers(
+                                        str(session_id),
+                                        student_id,
+                                        test_id
+                                    )
+                                    if evaluation:
+                                        st.success("Evaluation generated successfully!")
+                                        st.markdown("### Generated Evaluation")
+                                        for eval_item in evaluation['evaluations']:
+                                            st.markdown(f"#### Question {eval_item['question_number']}")
+                                            st.markdown(eval_item['evaluation'])
+                                            st.markdown("---")
+                                        st.rerun()
+                                    else:
+                                        st.error("Error generating evaluation.")
+
+    except Exception as e:
+        st.error(f"An error occurred while loading the evaluations: {str(e)}")
+        print(f"Error in display_evaluation_to_faculty: {str(e)}")

ui.py

@@ -1,111 +1,111 @@
-import streamlit as st
-from streamlit_option_menu import option_menu
-
-
-# Page Configuration
-st.set_page_config(page_title="Enhanced Navigation Demo", layout="wide")
-
-# Top Navigation Bar using option_menu
-selected = option_menu(
-    menu_title=None,
-    options=["Home", "Documentation", "Examples", "Community", "About"],
-    icons=["house", "book", "code", "people", "info-circle"],
-    menu_icon="cast",
-    default_index=0,
-    orientation="horizontal",
-    styles={
-        "container": {"padding": "0!important", "background-color": "#fafafa"},
-        "icon": {"color": "orange", "font-size": "25px"}, 
-        "nav-link": {
-            "font-size": "15px",
-            "text-align": "center",
-            "margin":"0px",
-            "--hover-color": "#eee",
-        },
-        "nav-link-selected": {"background-color": "#0083B8"},
-    }
-)
-
-# Sidebar Navigation
-with st.sidebar:
-    st.header("Navigation Menu")
-    
-    # Main Menu Items
-    selected_side = option_menu(
-        menu_title="Go to",
-        options=["Dashboard", "Analytics", "Reports", "Settings"],
-        icons=["speedometer2", "graph-up", "file-text", "gear"],
-        menu_icon="list",
-        default_index=0,
-    )
-    
-    # Expandable Reports Section
-    if selected_side == "Reports":
-        with st.expander("Reports", expanded=True):
-            st.button("Weekly Report")
-            st.button("Monthly Report")
-            st.button("Annual Report")
-
-# Main Content Area based on top navigation
-if selected == "Home":
-    st.title("Welcome to Home")
-    st.write("This is the home page content.")
-    
-    # Dashboard Content
-    st.header("Dashboard")
-    col1, col2, col3 = st.columns(3)
-    with col1:
-        st.metric("Sales", "$12,345", "+2.5%")
-    with col2:
-        st.metric("Users", "1,234", "-8%")
-    with col3:
-        st.metric("Conversion", "3.2%", "+1.2%")
-
-elif selected == "Documentation":
-    st.title("Documentation")
-    st.write("Documentation content goes here.")
-    
-elif selected == "Examples":
-    st.title("Examples")
-    st.write("Example content goes here.")
-    
-elif selected == "Community":
-    st.title("Community")
-    st.write("Community content goes here.")
-    
-elif selected == "About":
-    st.title("About")
-    st.write("About content goes here.")
-
-# Content based on sidebar selection
-if selected_side == "Analytics":
-    st.header("Analytics")
-    st.line_chart({"data": [1, 5, 2, 6, 2, 1]})
-elif selected_side == "Settings":
-    st.header("Settings")
-    st.toggle("Dark Mode")
-    st.toggle("Notifications")
-    st.slider("Volume", 0, 100, 50)
-
-# Footer
-st.markdown(
-    """
-    <style>
-    .footer {
-        position: fixed;
-        left: 0;
-        bottom: 0;
-        width: 100%;
-        background-color: #0E1117;
-        color: white;
-        text-align: center;
-        padding: 10px;
-        font-size: 14px;
-    }
-    </style>
-    <div class='footer'>
-        © 2024 Your App Name • Privacy Policy • Terms of Service
-    </div>
-    """,
-    unsafe_allow_html=True
+import streamlit as st
+from streamlit_option_menu import option_menu
+
+
+# Page Configuration
+st.set_page_config(page_title="Enhanced Navigation Demo", layout="wide")
+
+# Top Navigation Bar using option_menu
+selected = option_menu(
+    menu_title=None,
+    options=["Home", "Documentation", "Examples", "Community", "About"],
+    icons=["house", "book", "code", "people", "info-circle"],
+    menu_icon="cast",
+    default_index=0,
+    orientation="horizontal",
+    styles={
+        "container": {"padding": "0!important", "background-color": "#fafafa"},
+        "icon": {"color": "orange", "font-size": "25px"}, 
+        "nav-link": {
+            "font-size": "15px",
+            "text-align": "center",
+            "margin":"0px",
+            "--hover-color": "#eee",
+        },
+        "nav-link-selected": {"background-color": "#0083B8"},
+    }
+)
+
+# Sidebar Navigation
+with st.sidebar:
+    st.header("Navigation Menu")
+    
+    # Main Menu Items
+    selected_side = option_menu(
+        menu_title="Go to",
+        options=["Dashboard", "Analytics", "Reports", "Settings"],
+        icons=["speedometer2", "graph-up", "file-text", "gear"],
+        menu_icon="list",
+        default_index=0,
+    )
+    
+    # Expandable Reports Section
+    if selected_side == "Reports":
+        with st.expander("Reports", expanded=True):
+            st.button("Weekly Report")
+            st.button("Monthly Report")
+            st.button("Annual Report")
+
+# Main Content Area based on top navigation
+if selected == "Home":
+    st.title("Welcome to Home")
+    st.write("This is the home page content.")
+    
+    # Dashboard Content
+    st.header("Dashboard")
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        st.metric("Sales", "$12,345", "+2.5%")
+    with col2:
+        st.metric("Users", "1,234", "-8%")
+    with col3:
+        st.metric("Conversion", "3.2%", "+1.2%")
+
+elif selected == "Documentation":
+    st.title("Documentation")
+    st.write("Documentation content goes here.")
+    
+elif selected == "Examples":
+    st.title("Examples")
+    st.write("Example content goes here.")
+    
+elif selected == "Community":
+    st.title("Community")
+    st.write("Community content goes here.")
+    
+elif selected == "About":
+    st.title("About")
+    st.write("About content goes here.")
+
+# Content based on sidebar selection
+if selected_side == "Analytics":
+    st.header("Analytics")
+    st.line_chart({"data": [1, 5, 2, 6, 2, 1]})
+elif selected_side == "Settings":
+    st.header("Settings")
+    st.toggle("Dark Mode")
+    st.toggle("Notifications")
+    st.slider("Volume", 0, 100, 50)
+
+# Footer
+st.markdown(
+    """
+    <style>
+    .footer {
+        position: fixed;
+        left: 0;
+        bottom: 0;
+        width: 100%;
+        background-color: #0E1117;
+        color: white;
+        text-align: center;
+        padding: 10px;
+        font-size: 14px;
+    }
+    </style>
+    <div class='footer'>
+        © 2024 Your App Name • Privacy Policy • Terms of Service
+    </div>
+    """,
+    unsafe_allow_html=True
 )