Update app.py

app.py

@@ -8,7 +8,6 @@ import pickle
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 import scipy.special
 from tqdm import tqdm
-from tabulate import tabulate
 from sklearn.feature_extraction.text import TfidfVectorizer
 from multiprocessing import Pool, cpu_count
 from flask import Flask, request, jsonify
@@ -22,10 +21,9 @@ logger = logging.getLogger(__name__)
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 # Paths for saving artifacts
-MODEL_DIR = "./saved_models"  # Primary location in /app/saved_models
-FALLBACK_MODEL_DIR = "/tmp/saved_models"  # Fallback if ./saved_models fails
+MODEL_DIR = "./saved_models"
+FALLBACK_MODEL_DIR = "/tmp/saved_models"
 
-# Try to use the primary directory, fall back to /tmp if needed
 try:
     os.makedirs(MODEL_DIR, exist_ok=True)
     logger.info(f"Using model directory: {MODEL_DIR}")
@@ -35,13 +33,17 @@ except Exception as e:
     os.makedirs(FALLBACK_MODEL_DIR, exist_ok=True)
     chosen_model_dir = FALLBACK_MODEL_DIR
 
-# Update paths based on the chosen directory
+# Update paths
 UNIVERSAL_MODEL_PATH = os.path.join(chosen_model_dir, "universal_model")
 DETECTOR_MODEL_PATH = os.path.join(chosen_model_dir, "detector_model")
 TFIDF_PATH = os.path.join(chosen_model_dir, "tfidf_vectorizer.pkl")
 SKILL_TFIDF_PATH = os.path.join(chosen_model_dir, "skill_tfidf.pkl")
 QUESTION_ANSWER_PATH = os.path.join(chosen_model_dir, "question_to_answer.pkl")
 FAISS_INDEX_PATH = os.path.join(chosen_model_dir, "faiss_index.index")
+QUESTION_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "question_embeddings.pkl")
+COURSE_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "course_embeddings.pkl")
+JOB_SKILL_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "job_skill_embeddings.pkl")
+JOB_TITLE_EMBEDDINGS_PATH = os.path.join(chosen_model_dir, "job_title_embeddings.pkl")
 
 # Improved dataset loading with fallback
 def load_dataset(file_path, required_columns=[], fallback_data=None):
@@ -82,7 +84,7 @@ jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company
     'location': ['Remote', 'Islamabad', 'Karachi', 'Remote', 'Islamabad'],
     'required_skills': ['Linux, Kubernetes', 'AWS, Kubernetes', 'Python, Node.js', 'Python, SQL', 'Cybersecurity, Linux'],
     'job_description': ['DevOps role description', 'Cloud architecture position', 'Software engineering role', 'Data science position', 'Security analyst role'],
-    'level': ['Intermediate', 'Advanced', 'Intermediate', 'Intermediate', 'Intermediate']  # Added level for job recommendations
+    'level': ['Intermediate', 'Advanced', 'Intermediate', 'Intermediate', 'Intermediate']
 })
 
 # Validate questions_df
@@ -107,17 +109,23 @@ else:
     detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
     detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
 
-# Precompute Resources with Validation
-def resources_valid(saved_skills, current_skills):
-    return set(saved_skills) == set(current_skills)
+# Global variables for cached embeddings
+question_embeddings = None
+course_embeddings = None
+job_skill_embeddings = None
+job_title_embeddings = None
 
+# Precompute Resources with Caching
 def initialize_resources(user_skills):
-    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, question_embeddings, course_embeddings, job_skill_embeddings, job_title_embeddings
     
     user_skills_lower = [s.lower() for s in user_skills]
     needs_recompute = False
 
-    if all(os.path.exists(p) for p in [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH]):
+    # Check if cached resources exist
+    required_paths = [TFIDF_PATH, SKILL_TFIDF_PATH, QUESTION_ANSWER_PATH, FAISS_INDEX_PATH,
+                      QUESTION_EMBEDDINGS_PATH, COURSE_EMBEDDINGS_PATH, JOB_SKILL_EMBEDDINGS_PATH, JOB_TITLE_EMBEDDINGS_PATH]
+    if all(os.path.exists(p) for p in required_paths):
         try:
             with open(TFIDF_PATH, 'rb') as f:
                 tfidf_vectorizer = pickle.load(f)
@@ -126,6 +134,14 @@ def initialize_resources(user_skills):
             with open(QUESTION_ANSWER_PATH, 'rb') as f:
                 question_to_answer = pickle.load(f)
             faiss_index = faiss.read_index(FAISS_INDEX_PATH)
+            with open(QUESTION_EMBEDDINGS_PATH, 'rb') as f:
+                question_embeddings = pickle.load(f)
+            with open(COURSE_EMBEDDINGS_PATH, 'rb') as f:
+                course_embeddings = pickle.load(f)
+            with open(JOB_SKILL_EMBEDDINGS_PATH, 'rb') as f:
+                job_skill_embeddings = pickle.load(f)
+            with open(JOB_TITLE_EMBEDDINGS_PATH, 'rb') as f:
+                job_title_embeddings = pickle.load(f)
             
             if set(skill_tfidf.keys()) != set(user_skills_lower):
                 logger.info("Skill mismatch detected, recomputing resources")
@@ -144,23 +160,34 @@ def initialize_resources(user_skills):
         
         skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill]).toarray()[0] for skill in user_skills}
         question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
-        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True).cpu().numpy()
+        answers = list(question_to_answer.values())
+        question_embeddings = universal_model.encode(answers, batch_size=128, convert_to_tensor=True).cpu().numpy()
         
-        faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
-        faiss_index.add(answer_embeddings)
+        faiss_index = faiss.IndexFlatL2(question_embeddings.shape[1])
+        faiss_index.add(question_embeddings)
+        
+        # Precompute embeddings for courses and jobs
+        course_embeddings = universal_model.encode(courses_df['skills'].fillna("").tolist(), batch_size=128, convert_to_tensor=True).cpu().numpy()
+        job_field = 'required_skills' if 'required_skills' in jobs_df.columns else 'job_description'
+        job_skill_embeddings = universal_model.encode(jobs_df[job_field].fillna("").tolist(), batch_size=128, convert_to_tensor=True).cpu().numpy()
+        job_title_embeddings = universal_model.encode(jobs_df['job_title'].fillna("").tolist(), batch_size=128, convert_to_tensor=True).cpu().numpy()
         
         # Save resources
         with open(TFIDF_PATH, 'wb') as f: pickle.dump(tfidf_vectorizer, f)
         with open(SKILL_TFIDF_PATH, 'wb') as f: pickle.dump(skill_tfidf, f)
         with open(QUESTION_ANSWER_PATH, 'wb') as f: pickle.dump(question_to_answer, f)
         faiss.write_index(faiss_index, FAISS_INDEX_PATH)
+        with open(QUESTION_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(question_embeddings, f)
+        with open(COURSE_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(course_embeddings, f)
+        with open(JOB_SKILL_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(job_skill_embeddings, f)
+        with open(JOB_TITLE_EMBEDDINGS_PATH, 'wb') as f: pickle.dump(job_title_embeddings, f)
         universal_model.save(UNIVERSAL_MODEL_PATH)
         logger.info(f"Resources saved to {chosen_model_dir}")
 
-# Enhanced evaluation with batch processing
+# Enhanced evaluation with batch processing and parallelization
 def evaluate_response(args):
     try:
-        skill, user_answer, question = args
+        skill, user_answer, expected_answer, user_answer_embedding, expected_answer_embedding = args
         if not user_answer:
             return skill, 0.0, False
 
@@ -170,9 +197,7 @@ def evaluate_response(args):
         probs = scipy.special.softmax(logits, axis=1).tolist()[0]
         is_ai = probs[1] > 0.5
 
-        expected_answer = question_to_answer.get(question, "")
-        user_embeddings = universal_model.encode([user_answer, expected_answer], batch_size=32, convert_to_tensor=True)
-        score = util.pytorch_cos_sim(user_embeddings[0], user_embeddings[1]).item() * 100
+        score = util.pytorch_cos_sim(user_answer_embedding, expected_answer_embedding).item() * 100
 
         user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
         skill_vec = skill_tfidf.get(skill.lower(), np.zeros_like(user_tfidf))
@@ -184,7 +209,7 @@ def evaluate_response(args):
         logger.error(f"Evaluation error for {skill}: {e}")
         return skill, 0.0, False
 
-# Improved course recommendation with batch processing
+# Improved course recommendation with cached embeddings
 def recommend_courses(skills_to_improve, user_level, upgrade=False):
     try:
         if not skills_to_improve or courses_df.empty:
@@ -195,10 +220,8 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
         if 'completion_rate' not in courses_df:
             courses_df['completion_rate'] = 0.7
 
-        # Batch encode skills and courses
-        skill_embeddings = universal_model.encode(skills_to_improve, batch_size=32, convert_to_tensor=True)
-        course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), batch_size=32, convert_to_tensor=True)
-        similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()
+        skill_embeddings = universal_model.encode(skills_to_improve, batch_size=128, convert_to_tensor=True)
+        similarities = util.pytorch_cos_sim(skill_embeddings, torch.tensor(course_embeddings)).numpy()
 
         total_scores = 0.6 * similarities + 0.2 * courses_df['popularity'].values + 0.2 * courses_df['completion_rate'].values
 
@@ -215,29 +238,24 @@ def recommend_courses(skills_to_improve, user_level, upgrade=False):
         logger.error(f"Course recommendation error: {e}")
         return []
 
-# Enhanced job recommendation with fixed level handling
+# Enhanced job recommendation with cached embeddings
 def recommend_jobs(user_skills, user_level):
     try:
         if jobs_df.empty:
             return []
 
-        job_field = 'required_skills' if 'required_skills' in jobs_df.columns else 'job_description'
-        job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), batch_size=32, convert_to_tensor=True)
-        user_embedding = universal_model.encode(" ".join(user_skills), batch_size=32, convert_to_tensor=True)
-        skill_similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
+        user_embedding = universal_model.encode(" ".join(user_skills), batch_size=128, convert_to_tensor=True)
+        skill_similarities = util.pytorch_cos_sim(user_embedding, torch.tensor(job_skill_embeddings)).numpy()[0]
 
-        # Ensure level column exists and is a Series
         if 'level' not in jobs_df.columns:
             jobs_df['level'] = 'Intermediate'
-        level_col = jobs_df['level'].astype(str)  # Ensure it's a string Series
-
+        level_col = jobs_df['level'].astype(str)
         level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
         user_level_num = level_map.get(user_level, 1)
         level_scores = level_col.apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num)/2)
 
         location_pref = jobs_df.get('location', pd.Series(['Remote'] * len(jobs_df))).apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7)
-        industry_embeddings = universal_model.encode(jobs_df['job_title'].fillna(""), batch_size=32, convert_to_tensor=True)
-        industry_similarities = util.pytorch_cos_sim(user_embedding, industry_embeddings).numpy()[0]
+        industry_similarities = util.pytorch_cos_sim(user_embedding, torch.tensor(job_title_embeddings)).numpy()[0]
 
         total_job_scores = 0.5 * skill_similarities + 0.2 * level_scores + 0.1 * location_pref + 0.2 * industry_similarities
         top_job_indices = np.argsort(-total_job_scores)[:5]
@@ -292,22 +310,38 @@ def assess_skills():
         if len(user_questions) != len(user_skills):
             return jsonify({"error": f"Internal error: Number of selected questions ({len(user_questions)}) does not match number of skills ({len(user_skills)})."}), 500
 
+        # Batch encode all user answers and expected answers
+        user_answers = []
+        expected_answers = []
         user_responses = []
         for idx, row in user_questions.iterrows():
             answer = answers[idx]
             if not answer or answer.lower() == 'skip':
-                user_responses.append((row['Skill'], None, row['Question']))
+                user_responses.append((row['Skill'], None, None, None, None))
             else:
-                user_responses.append((row['Skill'], answer, row['Question']))
+                user_answers.append(answer)
+                expected_answer = question_to_answer.get(row['Question'], "")
+                expected_answers.append(expected_answer)
+                user_responses.append((row['Skill'], answer, expected_answer, None, None))
+
+        if user_answers:
+            all_embeddings = universal_model.encode(user_answers + expected_answers, batch_size=128, convert_to_tensor=True)
+            user_answer_embeddings = all_embeddings[:len(user_answers)]
+            expected_answer_embeddings = all_embeddings[len(user_answers):]
+            
+            for idx, (skill, answer, expected, _, _) in enumerate(user_responses):
+                if answer:
+                    user_responses[idx] = (skill, answer, expected, user_answer_embeddings[idx], expected_answer_embeddings[idx])
 
+        # Parallelize evaluation
         with Pool(processes=min(cpu_count(), 4)) as pool:
-            eval_args = [(skill, user_code, question) for skill, user_code, question in user_responses if user_code]
+            eval_args = [response for response in user_responses if response[1]]
             results = pool.map(evaluate_response, eval_args)
 
         user_scores = {}
         ai_flags = {}
         scores_list = []
-        skipped_questions = [f"{skill} ({question})" for skill, user_code, question in user_responses if user_code is None]
+        skipped_questions = [f"{skill} ({question})" for skill, user_code, question, _, _ in user_responses if not user_code]
         for skill, score, is_ai in results:
             if skill in user_scores:
                 user_scores[skill] = max(user_scores[skill], score)