Update app.py

app.py

@@ -1,21 +1,21 @@
 import os
-import numpy as np
+import pandas as pd
 import torch
 from sentence_transformers import SentenceTransformer, util
 import faiss
+import numpy as np
 import pickle
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import scipy.special
+from sklearn.feature_extraction.text import TfidfVectorizer
 from flask import Flask, request, jsonify
 import logging
-from pymongo import MongoClient
-import pandas as pd
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# Disable tokenizers parallelism
+# Disable tokenizers parallelism to avoid fork-related deadlocks
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 # Paths for saving artifacts
@@ -34,127 +34,282 @@ except Exception as e:
 # Update paths
 UNIVERSAL_MODEL_PATH = os.path.join(chosen_model_dir, "universal_model")
 DETECTOR_MODEL_PATH = os.path.join(chosen_model_dir, "detector_model")
+TFIDF_PATH = os.path.join(chosen_model_dir, "tfidf_vectorizer.pkl")
+SKILL_TFIDF_PATH = os.path.join(chosen_model_dir, "skill_tfidf.pkl")
+QUESTION_ANSWER_PATH = os.path.join(chosen_model_dir, "question_to_answer.pkl")
 FAISS_INDEX_PATH = os.path.join(chosen_model_dir, "faiss_index.index")
 ANSWER_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "answer_embeddings.pkl")
-COURSE_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "course_embeddings.pkl")
-JOB_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "job_embeddings.pkl")
+COURSE_SIMILARITY_PATH = os.path.join(chosen_model_dir, "course_similarity.pkl")
+JOB_SIMILARITY_PATH = os.path.join(chosen_model_dir, "job_similarity.pkl")
 
-# MongoDB connection (use the same URI as your Express app)
-MONGO_URI = "mongodb://localhost:27017/DMS"  # Replace with your MongoDB URI
-client = MongoClient(MONGO_URI)
-db = client.get_database()
-
-# Load models
-universal_model = SentenceTransformer(UNIVERSAL_MODEL_PATH) if os.path.exists(UNIVERSAL_MODEL_PATH) else SentenceTransformer("all-MiniLM-L6-v2")
-detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH) if os.path.exists(DETECTOR_MODEL_PATH) else AutoTokenizer.from_pretrained("roberta-base-openai-detector")
-detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH) if os.path.exists(DETECTOR_MODEL_PATH) else AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
-
-# Global variables
+# Global variables for precomputed data
+tfidf_vectorizer = None
+skill_tfidf = None
+question_to_answer = None
 faiss_index = None
 answer_embeddings = None
-course_embeddings = None
-job_embeddings = None
+course_similarity = None
+job_similarity = None
 
-# Load data from MongoDB
-def load_mongodb_data():
-    global answer_embeddings, course_embeddings, job_embeddings, faiss_index
+# Improved dataset loading with fallback
+def load_dataset(file_path, required_columns=[], additional_columns=['popularity', 'completion_rate'], fallback_data=None):
     try:
-        # Load questions from Generated_Skill-Based_Questions.csv (for now, keep as fallback; later, move to MongoDB)
-        questions_df = pd.read_csv("Generated_Skill-Based_Questions.csv")  # Replace with MongoDB query if stored
-        courses = list(db.courses.find())  # Fetch all courses
-        jobs = list(db.jobs.find())  # Fetch all jobs
-
-        # Precompute embeddings
+        df = pd.read_csv(file_path)
+        missing_required = [col for col in required_columns if col not in df.columns]
+        missing_additional = [col for col in additional_columns if col not in df.columns]
+        
+        # Handle missing required columns
+        if missing_required:
+            logger.warning(f"Required columns {missing_required} missing in {file_path}. Adding empty values.")
+            for col in missing_required:
+                df[col] = ""
+        
+        # Handle missing additional columns (popularity, completion_rate, etc.)
+        if missing_additional:
+            logger.warning(f"Additional columns {missing_additional} missing in {file_path}. Adding default values.")
+            for col in missing_additional:
+                if col == 'popularity':
+                    df[col] = 0.8  # Default value for popularity
+                elif col == 'completion_rate':
+                    df[col] = 0.7  # Default value for completion_rate
+                else:
+                    df[col] = 0.0  # Default for other additional columns
+        
+        # Ensure 'level' column has valid values (not empty)
+        if 'level' in df.columns:
+            df['level'] = df['level'].apply(lambda x: 'Intermediate' if pd.isna(x) or x.strip() == "" else x)
+        else:
+            logger.warning(f"'level' column missing in {file_path}. Adding default 'Intermediate'.")
+            df['level'] = 'Intermediate'
+        
+        return df
+    except ValueError as ve:
+        logger.error(f"ValueError loading {file_path}: {ve}. Using fallback data.")
+        if fallback_data is not None:
+            logger.info(f"Using fallback data for {file_path}")
+            return pd.DataFrame(fallback_data)
+        return None
+    except Exception as e:
+        logger.error(f"Error loading {file_path}: {e}. Using fallback data.")
+        if fallback_data is not None:
+            logger.info(f"Using fallback data for {file_path}")
+            return pd.DataFrame(fallback_data)
+        return None
+
+# Load datasets with fallbacks
+questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"], [], {
+    'Skill': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
+    'Question': ['Advanced Linux question', 'Advanced Git question', 'Basic Node.js question', 
+                 'Intermediate Python question', 'Basic Kubernetes question'],
+    'Answer': ['Linux answer', 'Git answer', 'Node.js answer', 'Python answer', 'Kubernetes answer']
+})
+
+courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"], ['popularity', 'completion_rate'], {
+    'skills': ['Linux', 'Git', 'Node.js', 'Python', 'Kubernetes'],
+    'course_title': ['Linux Admin', 'Git Mastery', 'Node.js Advanced', 'Python for Data', 'Kubernetes Basics'],
+    'Organization': ['Coursera', 'Udemy', 'Pluralsight', 'edX', 'Linux Foundation'],
+    'level': ['Intermediate', 'Intermediate', 'Advanced', 'Advanced', 'Intermediate'],
+    'popularity': [0.85, 0.9, 0.8, 0.95, 0.9],
+    'completion_rate': [0.65, 0.7, 0.6, 0.8, 0.75]
+})
+
+jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"], [], {
+    'job_title': ['DevOps Engineer', 'Cloud Architect', 'Software Engineer', 'Data Scientist', 'Security Analyst'],
+    'company_name': ['Tech Corp', 'Cloud Inc', 'Tech Solutions', 'Data Co', 'SecuriTech'],
+    'location': ['Remote', 'Islamabad', 'Karachi', 'Remote', 'Islamabad'],
+    'required_skills': ['Linux, Kubernetes', 'AWS, Kubernetes', 'Python, Node.js', 'Python, SQL', 'Cybersecurity, Linux'],
+    'job_description': ['DevOps role description', 'Cloud architecture position', 'Software engineering role', 'Data science position', 'Security analyst role'],
+    'level': ['Intermediate', 'Advanced', 'Intermediate', 'Intermediate', 'Intermediate']
+})
+
+# Validate questions_df
+if questions_df is None or questions_df.empty:
+    logger.error("questions_df is empty or could not be loaded. Exiting.")
+    exit(1)
+if not all(col in questions_df.columns for col in ["Skill", "Question", "Answer"]):
+    logger.error("questions_df is missing required columns. Exiting.")
+    exit(1)
+logger.info(f"questions_df loaded with {len(questions_df)} rows. Skills available: {list(questions_df['Skill'].unique())}")
+
+# Load or Initialize Models with Fallback
+def load_universal_model():
+    default_model = "all-MiniLM-L6-v2"
+    try:
+        if os.path.exists(UNIVERSAL_MODEL_PATH):
+            logger.info(f"Loading universal model from {UNIVERSAL_MODEL_PATH}")
+            return SentenceTransformer(UNIVERSAL_MODEL_PATH)
+        else:
+            logger.info(f"Loading universal model: {default_model}")
+            model = SentenceTransformer(default_model)
+            model.save(UNIVERSAL_MODEL_PATH)
+            return model
+    except Exception as e:
+        logger.error(f"Failed to load universal model {default_model}: {e}. Exiting.")
+        exit(1)
+
+universal_model = load_universal_model()
+
+if os.path.exists(DETECTOR_MODEL_PATH):
+    detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH)
+    detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH)
+else:
+    detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
+    detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
+
+# Load Precomputed Resources
+def load_precomputed_resources():
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
+    if all(os.path.exists(p) for p in [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH, ANSWER_EMBEDDINGS_PATH, COURSE_SIMILARITY_PATH, JOB_SIMILARITY_PATH]):
+        try:
+            with open(TFIDF_PATH, 'rb') as f: tfidf_vectorizer = pickle.load(f)
+            with open(SKILL_TFIDF_PATH, 'rb') as f: skill_tfidf = pickle.load(f)
+            with open(QUESTION_ANSWER_PATH, 'rb') as f: question_to_answer = pickle.load(f)
+            faiss_index = faiss.read_index(FAISS_INDEX_PATH)
+            with open(ANSWER_EMBEDDINGS_PATH, 'rb') as f: answer_embeddings = pickle.load(f)
+            with open(COURSE_SIMILARITY_PATH, 'rb') as f: course_similarity = pickle.load(f)
+            with open(JOB_SIMILARITY_PATH, 'rb') as f: job_similarity = pickle.load(f)
+            logger.info("Loaded precomputed resources successfully")
+        except Exception as e:
+            logger.error(f"Error loading precomputed resources: {e}")
+            precompute_resources()
+    else:
+        precompute_resources()
+
+# Precompute Resources Offline (to be run separately)
+def precompute_resources():
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings, course_similarity, job_similarity
+    logger.info("Precomputing resources offline")
+    try:
+        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
+        all_texts = questions_df['Answer'].tolist() + questions_df['Question'].tolist()
+        tfidf_vectorizer.fit(all_texts)
+        
+        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in questions_df['Skill'].unique()}
+        question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
         answer_embeddings = universal_model.encode(questions_df['Answer'].tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-        course_skills = [course['skills'] for course in courses]  # Adjust based on your Course schema
-        course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-        job_skills = [job['skills'] for job in jobs]  # Adjust based on your Job schema
-        job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu").cpu().numpy()
-
-        # Build FAISS index
+        
         faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
         faiss_index.add(answer_embeddings)
-
-        # Save precomputed data
-        with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
-        with open(COURSE_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(course_embeddings, f)
-        with open(JOB_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(job_embeddings, f)
+        
+        # Precompute course similarities
+        course_skills = courses_df['skills'].fillna("").tolist()
+        course_embeddings = universal_model.encode(course_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        skill_embeddings = universal_model.encode(questions_df['Skill'].unique().tolist(), batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        course_similarity = util.pytorch_cos_sim(skill_embeddings, course_embeddings).cpu().numpy()
+        
+        # Precompute job similarities
+        job_skills = jobs_df['required_skills'].fillna("").tolist()
+        job_embeddings = universal_model.encode(job_skills, batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")
+        job_similarity = util.pytorch_cos_sim(skill_embeddings, job_embeddings).cpu().numpy()
+        
+        # Save precomputed resources
+        with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
+        with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
+        with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
         faiss.write_index(faiss_index, FAISS_INDEX_PATH)
-        logger.info("Loaded and precomputed MongoDB data successfully")
+        with open(ANSWER_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(answer_embeddings, f)
+        with open(COURSE_SIMILARITY_PATH, 'wb') as f: pickle.dump(course_similarity, f)
+        with open(JOB_SIMILARITY_PATH, 'wb') as f: pickle.dump(job_similarity, f)
+        universal_model.save(UNIVERSAL_MODEL_PATH)
+        logger.info(f"Precomputed resources saved to {chosen_model_dir}")
     except Exception as e:
-        logger.error(f"Error loading MongoDB data: {e}")
+        logger.error(f"Error during precomputation: {e}")
         raise
 
-# Evaluate response (unchanged logic, but use MongoDB questions if stored)
+# Evaluation with precomputed data
 def evaluate_response(args):
-    skill, user_answer, question_idx = args
-    if not user_answer:
+    try:
+        skill, user_answer, question_idx = args
+        if not user_answer:
+            return skill, 0.0, False
+
+        inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
+        with torch.no_grad():
+            logits = detector_model(**inputs).logits
+        probs = scipy.special.softmax(logits, axis=1).tolist()[0]
+        is_ai = probs[1] > 0.5
+
+        user_embedding = universal_model.encode([user_answer], batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")[0]
+        expected_embedding = torch.tensor(answer_embeddings[question_idx])
+        score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
+
+        user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
+        skill_vec = skill_tfidf.get(skill.lower(), np.zeros_like(user_tfidf))
+        relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
+        score *= max(0.5, min(1.0, relevance))
+
+        return skill, round(max(0, score), 2), is_ai
+    except Exception as e:
+        logger.error(f"Evaluation error for {skill}: {e}")
         return skill, 0.0, False
 
-    inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
-    with torch.no_grad():
-        logits = detector_model(**inputs).logits
-    probs = scipy.special.softmax(logits, axis=1).tolist()[0]
-    is_ai = probs[1] > 0.5
-
-    user_embedding = universal_model.encode([user_answer], batch_size=128, convert_to_tensor=True, device="cuda" if torch.cuda.is_available() else "cpu")[0]
-    expected_embedding = torch.tensor(answer_embeddings[question_idx])
-    score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
-    return skill, round(max(0, score), 2), is_ai
-
-# Recommend courses from MongoDB
+# Course recommendation with precomputed similarity
 def recommend_courses(skills_to_improve, user_level, upgrade=False):
-    if not skills_to_improve or not course_embeddings:
-        return []
-    
-    skill_indices = [i for i, skill in enumerate(questions_df['Skill'].unique()) if skill in skills_to_improve]
-    if not skill_indices:
+    try:
+        if not skills_to_improve or courses_df.empty:
+            logger.info("No skills to improve or courses_df is empty.")
+            return []
+
+        skill_indices = [list(questions_df['Skill'].unique()).index(skill) for skill in skills_to_improve if skill in questions_df['Skill'].unique()]
+        if not skill_indices:
+            logger.info("No matching skill indices found.")
+            return []
+
+        similarities = course_similarity[skill_indices]
+        # Use default arrays to avoid KeyError
+        popularity = courses_df['popularity'].values if 'popularity' in courses_df else np.full(len(courses_df), 0.8)
+        completion_rate = courses_df['completion_rate'].values if 'completion_rate' in courses_df else np.full(len(courses_df), 0.7)
+        total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * popularity + 0.2 * completion_rate
+
+        target_level = 'Advanced' if upgrade else user_level
+        idx = np.argsort(-total_scores)[:5]
+        candidates = courses_df.iloc[idx]
+        
+        # Filter by level, but fallback to all courses if none match
+        filtered_candidates = candidates[candidates['level'].str.contains(target_level, case=False, na=False)]
+        if filtered_candidates.empty:
+            logger.warning(f"No courses found for level {target_level}. Returning top courses regardless of level.")
+            filtered_candidates = candidates
+        
+        return filtered_candidates[['course_title', 'Organization']].values.tolist()[:3]
+    except Exception as e:
+        logger.error(f"Course recommendation error: {e}")
         return []
 
-    similarities = util.pytorch_cos_sim(
-        torch.tensor(universal_model.encode(questions_df['Skill'].unique()[skill_indices].tolist(), batch_size=128)),
-        torch.tensor(course_embeddings)
-    ).cpu().numpy()
-    
-    courses = list(db.courses.find())
-    popularity = [course.get('popularity', 0.8) for course in courses]
-    completion_rate = [course.get('completion_rate', 0.7) for course in courses]
-    total_scores = 0.6 * np.max(similarities, axis=0) + 0.2 * np.array(popularity) + 0.2 * np.array(completion_rate)
-
-    target_level = 'Advanced' if upgrade else user_level
-    idx = np.argsort(-total_scores)[:5]
-    candidates = [courses[i] for i in idx]
-    filtered_candidates = [c for c in candidates if target_level.lower() in c.get('level', 'Intermediate').lower()]
-    return filtered_candidates[:3] if filtered_candidates else candidates[:3]
-
-# Recommend jobs from MongoDB
+# Job recommendation with precomputed similarity
 def recommend_jobs(user_skills, user_level):
-    if not job_embeddings:
-        return []
-    
-    skill_indices = [i for i, skill in enumerate(questions_df['Skill'].unique()) if skill in user_skills]
-    if not skill_indices:
+    try:
+        if jobs_df.empty:
+            return []
+
+        skill_indices = [list(questions_df['Skill'].unique()).index(skill) for skill in user_skills if skill in questions_df['Skill'].unique()]
+        if not skill_indices:
+            return []
+
+        similarities = job_similarity[skill_indices]
+        total_scores = 0.5 * np.max(similarities, axis=0)
+
+        if 'level' not in jobs_df.columns:
+            jobs_df['level'] = 'Intermediate'
+        level_col = jobs_df['level'].astype(str)
+        level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
+        user_level_num = level_map.get(user_level, 1)
+        level_scores = level_col.apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num)/2)
+
+        location_pref = jobs_df.get('location', pd.Series(['Remote'] * len(jobs_df))).apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7)
+        total_job_scores = total_scores + 0.2 * level_scores + 0.1 * location_pref
+        top_job_indices = np.argsort(-total_job_scores)[:5]
+
+        return [(jobs_df.iloc[i]['job_title'], jobs_df.iloc[i]['company_name'], 
+                jobs_df.iloc[i].get('location', 'Remote')) for i in top_job_indices]
+    except Exception as e:
+        logger.error(f"Job recommendation error: {e}")
         return []
 
-    similarities = util.pytorch_cos_sim(
-        torch.tensor(universal_model.encode(questions_df['Skill'].unique()[skill_indices].tolist(), batch_size=128)),
-        torch.tensor(job_embeddings)
-    ).cpu().numpy()
-    
-    jobs = list(db.jobs.find())
-    level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
-    user_level_num = level_map.get(user_level, 1)
-    level_scores = [1 - abs(level_map.get(job.get('level', 'Intermediate'), 1) - user_level_num) / 2 for job in jobs]
-    location_pref = [1.0 if job.get('location', 'Remote') in ['Islamabad', 'Karachi'] else 0.7 for job in jobs]
-    total_job_scores = 0.5 * np.max(similarities, axis=0) + 0.2 * np.array(level_scores) + 0.1 * np.array(location_pref)
-
-    top_job_indices = np.argsort(-total_job_scores)[:5]
-    return [(jobs[i]['jobTitle'], jobs[i]['companyName'], jobs[i].get('location', 'Remote')) for i in top_job_indices]
-
-# Flask app setup
+# Flask application setup
 app = Flask(__name__)
 
-@app.route('/health')
+@app.route('/')
 def health_check():
     return jsonify({"status": "active", "model_dir": chosen_model_dir})
 
@@ -172,10 +327,8 @@ def assess_skills():
         if len(answers) != len(user_skills):
             return jsonify({"error": "Answers count must match skills count"}), 400
 
-        load_mongodb_data()  # Load and precompute MongoDB data
+        load_precomputed_resources()  # Load precomputed resources before processing
 
-        # Generate questions (for now, use CSV as fallback; move to MongoDB later)
-        questions_df = pd.read_csv("Generated_Skill-Based_Questions.csv")
         user_questions = []
         for skill in user_skills:
             skill_questions = questions_df[questions_df['Skill'] == skill]
@@ -235,7 +388,7 @@ def assess_skills():
                 "weak_skills": weak_skills,
                 "skipped_questions": skipped_questions
             },
-            "recommended_courses": [{"course_title": c['title'], "organization": c.get('organization', 'Unknown')} for c in courses],
+            "recommended_courses": courses[:3],
             "recommended_jobs": jobs[:5]
         })
     except Exception as e: