Update app.py

app.py

@@ -1,177 +1,288 @@
-from fastapi import FastAPI, HTTPException
-from pydantic import BaseModel
-from typing import Optional, Dict, List
 import pandas as pd
-from sentence_transformers import SentenceTransformer
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-import time
+import torch
+from sentence_transformers import SentenceTransformer, util
+import faiss
+import numpy as np
 import os
+import pickle
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import scipy.special
+from tqdm import tqdm
+from tabulate import tabulate
+from sklearn.feature_extraction.text import TfidfVectorizer
+from multiprocessing import Pool, cpu_count
+from flask import Flask, request, jsonify
 
-# Set cache directory
-os.environ["HF_HOME"] = "/app/cache"
-os.environ["TRANSFORMERS_CACHE"] = "/app/cache"
-
-app = FastAPI()
-
-# Load datasets
-DATA_DIR = "/app/data/"
-job_df = pd.read_csv(os.path.join(DATA_DIR, "Updated_Job_Posting_Dataset.csv"), encoding="latin1")
-course_df = pd.read_csv(os.path.join(DATA_DIR, "coursera_course_dataset_v2_no_null.csv"))
-coding_df = pd.read_csv(os.path.join(DATA_DIR, "Software Questions.csv"), encoding="latin1")
-
-# Preprocess datasets
-coding_df = coding_df.rename(columns={
-    'Question': 'question',
-    'Answer': 'solutions',
-    'Category': 'category',
-    'Difficulty': 'difficulty'
-})
-coding_df.dropna(subset=['question', 'solutions', 'category', 'difficulty'], inplace=True)
-job_df.rename(columns={'company_name': 'company', 'required_skills': 'skills'}, inplace=True)
-course_df.rename(columns={'Title': 'course_title', 'Skills': 'skills'}, inplace=True)
-job_df["job_description"] = job_df["job_description"].fillna("")
-
-# Load BERT model and vectorizer
-bert_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
-vectorizer = TfidfVectorizer()
-
-# Pydantic models for request bodies
-class ChallengeRequest(BaseModel):
-    skills: List[str]
-    difficulty: Optional[str] = None
-
-class AssessmentRequest(BaseModel):
-    name: str
-    skills: List[str]
-    answers: Optional[Dict[str, Dict[str, str]]] = None
-
-# Get coding challenges
-def get_coding_challenges(categories: List[str], num_questions=5, difficulty: Optional[str] = None):
-    skill_challenges = {}
-    for category in categories:
-        relevant = coding_df[coding_df["category"].str.contains(category, case=False, na=False)]
-        if difficulty:
-            relevant = relevant[relevant["difficulty"].str.lower() == difficulty.lower()]
-        if not relevant.empty:
-            skill_challenges[category] = relevant.sample(min(num_questions, len(relevant)))[["question", "solutions", "difficulty"]].to_dict(orient="records")
-        else:
-            skill_challenges[category] = []
-    return skill_challenges
-
-# Evaluate coding answers
-def evaluate_coding_with_time(user_code, correct_code, start_time):
-    end_time = time.time()
-    execution_time = end_time - start_time
-    vectorized = vectorizer.fit_transform([user_code, correct_code])
-    similarity = cosine_similarity(vectorized)[0][1] * 100
-    if execution_time > 120:
-        similarity -= (execution_time - 120) * 0.1
-    return round(max(similarity, 0), 2)
-
-# Assign proficiency level
-def get_proficiency_level(score):
-    if score >= 80:
-        return "Expert"
-    elif score >= 50:
-        return "Intermediate"
+# Paths for saving artifacts
+MODEL_DIR = "./saved_models"
+UNIVERSAL_MODEL_PATH = os.path.join(MODEL_DIR, "universal_model")
+DETECTOR_MODEL_PATH = os.path.join(MODEL_DIR, "detector_model")
+TFIDF_PATH = os.path.join(MODEL_DIR, "tfidf_vectorizer.pkl")
+SKILL_TFIDF_PATH = os.path.join(MODEL_DIR, "skill_tfidf.pkl")
+QUESTION_ANSWER_PATH = os.path.join(MODEL_DIR, "question_to_answer.pkl")
+FAISS_INDEX_PATH = os.path.join(MODEL_DIR, "faiss_index.index")
+
+os.makedirs(MODEL_DIR, exist_ok=True)
+
+# Load Datasets
+def load_dataset(file_path, required_columns=[]):
+    try:
+        df = pd.read_csv(file_path)
+        for col in required_columns:
+            if col not in df.columns:
+                print(f"⚠ Warning: Column '{col}' missing in {file_path}. Using default values.")
+                df[col] = "" if col != 'level' else 'Intermediate'
+        return df
+    except FileNotFoundError:
+        print(f"❌ Error: Dataset not found at {file_path}. Exiting.")
+        return None
+
+user_df = load_dataset("Updated_User_Profile_Dataset.csv", ["name", "skills", "level"])
+questions_df = load_dataset("Generated_Skill-Based_Questions.csv", ["Skill", "Question", "Answer"])
+courses_df = load_dataset("coursera_course_dataset_v2_no_null.csv", ["skills", "course_title", "Organization", "level"])
+jobs_df = load_dataset("Updated_Job_Posting_Dataset.csv", ["job_title", "company_name", "location", "required_skills", "job_description"])
+
+# Simulate courses_df with relevant skills
+if courses_df is None or 'skills' not in courses_df.columns or courses_df['skills'].str.strip().eq('').all():
+    courses_df = pd.DataFrame({
+        'skills': ['Docker', 'Jenkins', 'Azure', 'Cybersecurity'],
+        'course_title': ['Docker Mastery', 'Jenkins CI/CD', 'Azure Fundamentals', 'Cybersecurity Basics'],
+        'Organization': ['Udemy', 'Coursera', 'Microsoft', 'edX'],
+        'level': ['Intermediate', 'Intermediate', 'Intermediate', 'Advanced'],
+        'popularity': [0.9, 0.85, 0.95, 0.8],
+        'completion_rate': [0.7, 0.65, 0.8, 0.6]
+    })
+
+# Load or Initialize Models
+if os.path.exists(UNIVERSAL_MODEL_PATH):
+    universal_model = SentenceTransformer(UNIVERSAL_MODEL_PATH)
+else:
+    universal_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+if os.path.exists(DETECTOR_MODEL_PATH):
+    detector_tokenizer = AutoTokenizer.from_pretrained(DETECTOR_MODEL_PATH)
+    detector_model = AutoModelForSequenceClassification.from_pretrained(DETECTOR_MODEL_PATH)
+else:
+    detector_tokenizer = AutoTokenizer.from_pretrained("roberta-base-openai-detector")
+    detector_model = AutoModelForSequenceClassification.from_pretrained("roberta-base-openai-detector")
+
+# Precompute Resources with Validation
+def resources_valid(saved_skills, current_skills):
+    return set(saved_skills) == set(current_skills)
+
+def initialize_resources(user_skills):
+    global tfidf_vectorizer, skill_tfidf, question_to_answer, faiss_index, answer_embeddings
+    if (os.path.exists(TFIDF_PATH) and os.path.exists(SKILL_TFIDF_PATH) and 
+        os.path.exists(QUESTION_ANSWER_PATH) and os.path.exists(FAISS_INDEX_PATH)):
+        with open(TFIDF_PATH, 'rb') as f:
+            tfidf_vectorizer = pickle.load(f)
+        with open(SKILL_TFIDF_PATH, 'rb') as f:
+            skill_tfidf = pickle.load(f)
+        with open(QUESTION_ANSWER_PATH, 'rb') as f:
+            question_to_answer = pickle.load(f)
+        faiss_index = faiss.read_index(FAISS_INDEX_PATH)
+        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
+        
+        if not resources_valid(skill_tfidf.keys(), [s.lower() for s in user_skills]):
+            print("⚠ Saved skill TF-IDF mismatch detected. Recomputing resources.")
+            tfidf_vectorizer = TfidfVectorizer(stop_words='english')
+            all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
+            tfidf_vectorizer.fit(all_texts)
+            skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
+            question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
+            answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
+            faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
+            faiss_index.add(answer_embeddings)
     else:
-        return "Beginner"
+        tfidf_vectorizer = TfidfVectorizer(stop_words='english')
+        all_texts = user_skills + questions_df['Answer'].fillna("").tolist() + questions_df['Question'].tolist()
+        tfidf_vectorizer.fit(all_texts)
+        skill_tfidf = {skill.lower(): tfidf_vectorizer.transform([skill.lower()]).toarray()[0] for skill in user_skills}
+        question_to_answer = dict(zip(questions_df['Question'], questions_df['Answer']))
+        answer_embeddings = universal_model.encode(list(question_to_answer.values()), convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
+        faiss_index = faiss.IndexFlatL2(answer_embeddings.shape[1])
+        faiss_index.add(answer_embeddings)
 
-# Recommend courses
-def recommend_courses(weak_skills):
-    if not weak_skills:
-        return []
-    courses = course_df[course_df['skills'].str.contains('|'.join(weak_skills), case=False, na=False)]
-    return courses[['course_title', 'Organization']].head(5).to_dict(orient="records")
+        with open(TFIDF_PATH, 'wb') as f:
+            pickle.dump(tfidf_vectorizer, f)
+        with open(SKILL_TFIDF_PATH, 'wb') as f:
+            pickle.dump(skill_tfidf, f)
+        with open(QUESTION_ANSWER_PATH, 'wb') as f:
+            pickle.dump(question_to_answer, f)
+        faiss.write_index(faiss_index, FAISS_INDEX_PATH)
+        universal_model.save_pretrained(UNIVERSAL_MODEL_PATH)
+        detector_model.save_pretrained(DETECTOR_MODEL_PATH)
+        detector_tokenizer.save_pretrained(DETECTOR_MODEL_PATH)
+        print(f"Models and resources saved to {MODEL_DIR}")
+
+# Evaluate Responses
+def evaluate_response(args):
+    skill, user_answer, question = args
+    if not user_answer:
+        return skill, 0, False
+    
+    inputs = detector_tokenizer(user_answer, return_tensors="pt", truncation=True, max_length=512)
+    with torch.no_grad():
+        logits = detector_model(**inputs).logits
+    probs = scipy.special.softmax(logits, axis=1).tolist()[0]
+    is_ai_generated = probs[1] > 0.5
 
-# Recommend jobs
-def recommend_jobs(skills):
-    if not skills:
+    user_embedding = universal_model.encode(user_answer, convert_to_tensor=True)
+    expected_answer = question_to_answer.get(question, "")
+    expected_embedding = universal_model.encode(expected_answer, convert_to_tensor=True)
+    score = util.pytorch_cos_sim(user_embedding, expected_embedding).item() * 100
+
+    user_tfidf = tfidf_vectorizer.transform([user_answer]).toarray()[0]
+    skill_lower = skill.lower()
+    skill_vec = skill_tfidf.get(skill_lower, tfidf_vectorizer.transform([skill_lower]).toarray()[0])
+    skill_relevance = np.dot(user_tfidf, skill_vec) / (np.linalg.norm(user_tfidf) * np.linalg.norm(skill_vec) + 1e-10)
+    penalty = min(1.0, max(0.5, skill_relevance))
+    score *= penalty
+
+    return skill, round(max(0, score), 2), is_ai_generated
+
+# Recommend Courses
+def recommend_courses(skills_to_improve, user_level, upgrade=False):
+    if not skills_to_improve:
         return []
-    job_df["job_embeddings"] = job_df["job_description"].apply(lambda x: bert_model.encode(str(x)))
-    user_embedding = bert_model.encode(" ".join(skills))
-    job_df["BERT_Similarity"] = job_df["job_embeddings"].apply(lambda x: cosine_similarity([x], [user_embedding])[0][0])
-    top_jobs = job_df.sort_values(by="BERT_Similarity", ascending=False).head(5)
-    return top_jobs[["job_title", "company", "location", "BERT_Similarity"]].to_dict(orient="records")
-
-@app.get("/")
-def read_root():
-    return {"message": "Skill Assessment API"}
-
-# POST endpoint for fetching challenges
-@app.post("/challenges")
-def get_user_challenges(request: ChallengeRequest):
-    skills = request.skills
-    difficulty = request.difficulty
     
-    if not skills:
-        raise HTTPException(status_code=400, detail="Skills list cannot be empty")
-
-    challenges = get_coding_challenges(skills, difficulty=difficulty)
-
-    # Return only questions and difficulty (exclude solutions for the user)
-    return {
-        "challenges": {
-            category: [
-                {"question": challenge["question"], "difficulty": challenge["difficulty"]}
-                for challenge in challenge_list
-            ]
-            for category, challenge_list in challenges.items()
-        }
-    }
+    skill_embeddings = universal_model.encode(skills_to_improve, convert_to_tensor=True)
+    course_embeddings = universal_model.encode(courses_df['skills'].fillna(""), convert_to_tensor=True)
+    bert_similarities = util.pytorch_cos_sim(skill_embeddings, course_embeddings).numpy()
+
+    collab_scores = []
+    for skill in skills_to_improve:
+        overlap = sum(1 for user_skills_str in user_df['skills'] if pd.notna(user_skills_str) and skill.lower() in user_skills_str.lower())
+        collab_scores.append(overlap / len(user_df))
+    collab_similarities = np.array([collab_scores]).repeat(len(courses_df), axis=0).T
 
-# POST endpoint for assessing answers
-@app.post("/assess")
-def assess_skills(user_input: AssessmentRequest):
-    user_name = user_input.name
-    user_skills = user_input.skills
+    popularity = courses_df['popularity'].fillna(0.5).to_numpy()
+    completion = courses_df['completion_rate'].fillna(0.5).to_numpy()
+    total_scores = (0.6 * bert_similarities + 0.2 * collab_similarities + 0.1 * popularity + 0.1 * completion)
 
-    if not user_skills:
-        raise HTTPException(status_code=400, detail="Skills list cannot be empty")
+    recommended_courses = []
+    target_level = 'Advanced' if upgrade else user_level
+    for i, skill in enumerate(skills_to_improve):
+        top_indices = total_scores[i].argsort()[-5:][::-1]
+        candidates = courses_df.iloc[top_indices]
+        candidates = candidates[candidates['skills'].str.lower() == skill.lower()]
+        if candidates.empty:
+            candidates = courses_df.iloc[top_indices]
+        candidates.loc[:, "level_match"] = candidates['level'].apply(lambda x: 1 if x == target_level else 0.8 if abs({'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[x] - {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}[user_level]) <= 1 else 0.5)
+        level_filtered = candidates.sort_values(by="level_match", ascending=False)
+        recommended_courses.extend(level_filtered[['course_title', 'Organization']].values.tolist()[:3])
+    return list(dict.fromkeys(tuple(course) for course in recommended_courses if course[0].strip()))
 
-    challenges = get_coding_challenges(user_skills)
+# Recommend Jobs
+def recommend_jobs(user_skills, user_level):
+    job_field = 'required_skills' if 'required_skills' in jobs_df.columns and not jobs_df['required_skills'].str.strip().eq('').all() else 'job_description'
+    job_embeddings = universal_model.encode(jobs_df[job_field].fillna(""), convert_to_tensor=True)
+    user_embedding = universal_model.encode(" ".join(user_skills), convert_to_tensor=True)
+    skill_similarities = util.pytorch_cos_sim(user_embedding, job_embeddings).numpy()[0]
+
+    level_map = {'Beginner': 0, 'Intermediate': 1, 'Advanced': 2}
+    user_level_num = level_map[user_level]
+    exp_match = jobs_df['level'].fillna('Intermediate').apply(lambda x: 1 - abs(level_map.get(x, 1) - user_level_num) / 2) if 'level' in jobs_df.columns else np.ones(len(jobs_df)) * 0.5
+    location_pref = jobs_df['location'].apply(lambda x: 1.0 if x in ['Islamabad', 'Karachi'] else 0.7).to_numpy()
+    industry_embeddings = universal_model.encode(jobs_df['job_title'].fillna(""), convert_to_tensor=True)
+    industry_similarities = util.pytorch_cos_sim(user_embedding, industry_embeddings).numpy()[0]
+
+    total_job_scores = (0.5 * skill_similarities + 0.2 * exp_match + 0.1 * location_pref + 0.2 * industry_similarities)
+    top_job_indices = total_job_scores.argsort()[-5:][::-1]
+    return [(jobs_df.iloc[idx]['job_title'], jobs_df.iloc[idx]['company_name'], jobs_df.iloc[idx]['location']) for idx in top_job_indices]
+
+# Main API Endpoint
+app = Flask(__name__)
+
+@app.route('/assess', methods=['POST'])
+def assess_skills():
+    data = request.get_json()
+    if not data or 'user_index' not in data or 'answers' not in data:
+        return jsonify({"error": "Invalid input. Provide 'user_index' and 'answers' in JSON body."}), 400
+
+    user_index = int(data['user_index'])
+    if user_index < 0 or user_index >= len(user_df):
+        return jsonify({"error": "Invalid user index."}), 400
+
+    user_text = user_df.loc[user_index, 'skills']
+    user_skills = [skill.strip() for skill in user_text.split(",") if skill.strip()] if isinstance(user_text, str) else ["Python", "SQL"]
+    user_name = user_df.loc[user_index, 'name']
+    user_level = user_df.loc[user_index, 'level'] if 'level' in user_df.columns and pd.notna(user_df.loc[user_index, 'level']) else 'Intermediate'
+
+    initialize_resources(user_skills)
+
+    filtered_questions = questions_df[questions_df['Skill'].isin(user_skills)]
+    if filtered_questions.empty:
+        return jsonify({"error": "No matching questions found!"}), 500
     
+    user_questions = []
+    for skill in user_skills:
+        skill_questions = filtered_questions[filtered_questions['Skill'] == skill]
+        if not skill_questions.empty:
+            user_questions.append(skill_questions.sample(1).iloc[0])
+    user_questions = pd.DataFrame(user_questions)
+
+    if len(user_questions) != 4:
+        return jsonify({"error": "Not enough questions for all skills!"}), 500
+
+    answers = data['answers']
+    if len(answers) != 4:
+        return jsonify({"error": "Please provide exactly 4 answers."}), 400
+
+    user_responses = []
+    for idx, row in user_questions.iterrows():
+        answer = answers[idx]
+        if not answer or answer.lower() == 'skip':
+            user_responses.append((row['Skill'], None, row['Question']))
+        else:
+            user_responses.append((row['Skill'], answer, row['Question']))
+
+    with Pool(cpu_count()) as pool:
+        eval_args = [(skill, user_code, question) for skill, user_code, question in user_responses if user_code]
+        results = pool.map(evaluate_response, eval_args)
+
     user_scores = {}
-    for skill, challenge_list in challenges.items():
-        if not challenge_list:
-            user_scores[skill] = 0
-            continue
-        
-        total_score = 0
-        num_questions = len(challenge_list)
-        
-        if user_input.answers and skill in user_input.answers:
-            for challenge in challenge_list:
-                question = challenge["question"]
-                if question in user_input.answers[skill]:
-                    start_time = time.time() - 10  # Simulate execution time
-                    user_code = user_input.answers[skill][question]
-                    correct_code = challenge["solutions"]
-                    score = evaluate_coding_with_time(user_code, correct_code, start_time)
-                    total_score += score
-                else:
-                    total_score += 0
+    ai_flags = {}
+    scores_list = []
+    skipped_questions = [f"{skill} ({question})" for skill, user_code, question in user_responses if user_code is None]
+    for skill, score, is_ai in results:
+        if skill in user_scores:
+            user_scores[skill] = max(user_scores[skill], score)
+            ai_flags[skill] = ai_flags[skill] or is_ai
         else:
-            total_score = 50 * num_questions  # Default score for unattempted questions
-        
-        user_scores[skill] = round(total_score / num_questions, 2)
+            user_scores[skill] = score
+            ai_flags[skill] = is_ai
+        scores_list.append(score)
+
+    mean_score = np.mean(scores_list) if scores_list else 50
+    dynamic_threshold = max(40, mean_score)
+    weak_skills = [skill for skill, score in user_scores.items() if score < dynamic_threshold]
+
+    assessment_results = [
+        (skill, f"{'■' * int(score//10)}{'-' * (10 - int(score//10))}", f"{score:.2f}%", "AI-Generated" if ai_flags[skill] else "Human-Written")
+        for skill, score in user_scores.items()
+    ]
+    assessment_output = tabulate(assessment_results, headers=["Skill", "Progress", "Score", "Origin"], tablefmt="grid")
+    if skipped_questions:
+        assessment_output += f"\nSkipped Questions: {skipped_questions}"
+    assessment_output += f"\nMean Score: {mean_score:.2f}, Dynamic Threshold: {dynamic_threshold:.2f}"
+    assessment_output += f"\nWeak Skills: {weak_skills if weak_skills else 'None'}"
 
-    proficiency_levels = {skill: get_proficiency_level(score) for skill, score in user_scores.items()}
-    weak_skills = [skill for skill, level in proficiency_levels.items() if level in ["Beginner", "Intermediate"]]
+    skills_to_recommend = weak_skills if weak_skills else user_skills
+    upgrade_flag = not weak_skills
+    recommended_courses = recommend_courses(skills_to_recommend, user_level, upgrade=upgrade_flag)
+    courses_output = tabulate(recommended_courses, headers=["Course", "Organization"], tablefmt="grid") if recommended_courses else "None"
 
-    courses = recommend_courses(weak_skills)
-    jobs = recommend_jobs(user_skills)
+    recommended_jobs = recommend_jobs(user_skills, user_level)
+    jobs_output = tabulate(recommended_jobs, headers=["Job Title", "Company", "Location"], tablefmt="grid")
 
-    return {
-        "name": user_name,
-        "skills": user_skills,
-        "scores": user_scores,
-        "proficiency_levels": proficiency_levels,
-        "recommended_courses": courses,
-        "recommended_jobs": jobs
+    response = {
+        "user_info": f"User: {user_name}\nSkills: {user_skills}\nLevel: {user_level}",
+        "assessment_results": assessment_output,
+        "recommended_courses": courses_output,
+        "recommended_jobs": jobs_output
     }
+    return jsonify(response)
 
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=7860)
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=7860)