Spaces:
Runtime error
Runtime error
File size: 1,892 Bytes
aadb779 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
import re
import pandas as pd
import numpy as np
from typing import Tuple, List
from sklearn.metrics import confusion_matrix
import seaborn as sns
PROGRAM = "Program"
def clean_text(text):
text_input = re.sub('[^a-zA-Z1-9]+', ' ', str(text))
output = re.sub(r'\d+', '', text_input)
return output.lower().strip()
def get_num_courses_per_program():
df = pd.read_csv('program_courses.csv')
return df.groupby([PROGRAM])[PROGRAM].count()
def load_data(num_majors=20, include_majors=[]) -> Tuple[List[str], np.ndarray]:
"""
Loads and preprocesses `course_sentences` data.
"""
courses = pd.read_csv("course_sentences.csv").drop(["course"], axis=1).dropna()
descriptions = pd.read_csv("program_descriptions.csv").rename(columns={"description": "sentence"}).dropna()
df = pd.concat([courses, descriptions], axis=0, ignore_index=True)
majors = list(df.groupby("program").count().sort_values(by=["sentence"], ascending=False).index)
majors = include_majors + majors
majors = majors[:num_majors]
df = df[df["program"].isin(majors)]
sentences = list(df["sentence"])
labels = np.array(df["program"])
return sentences, labels
def plot_confusion_matrix(y_true:List[str], y_pred:List[str], classes:List[str]):
"""Plots a confusion matrix"""
cm = confusion_matrix(y_true, y_pred, labels=classes)
cm_df=pd.DataFrame(data=cm, index=classes, columns=classes)
sns.heatmap(cm_df, annot=True)
def get_recommendations(probs:np.ndarray, labels:List[str], n=5) -> List[List[str]]:
"""
Args:
`probs`: predictions array of shape (n_inputs,n_classes)
`labels`: class labels of shape (n_classes,)
`n`: number of recommendations
Returns:
Top labels based on a probability distribution
"""
np_labels = np.array(labels)
return np_labels[(-probs).argsort(-1)[:,:n]]
|