Create recommender.py

recommender.py

@@ -0,0 +1,265 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from ast import literal_eval
+import gc
+from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
+from sklearn.preprocessing import MinMaxScaler
+from sklearn.metrics.pairwise import cosine_similarity
+# import seaborn as sns
+from collections import Counter
+
+import mlflow
+
+
+def init_mlflow():
+    mlflow.set_tracking_uri("http://0.0.0.0:8889")
+    mlflow.set_experiment("Default")
+    mlflow.start_run()
+    mlflow.sklearn.autolog()
+
+
+def load_data():
+    credits_df = pd.read_csv('./datasets/credits.csv')
+    keywords_df = pd.read_csv('./datasets/keywords.csv')
+    links_df = pd.read_csv('./datasets/links_small.csv')
+    movies_df = pd.read_csv('./datasets/movies_metadata.csv')
+    ratings_df = pd.read_csv('./datasets/ratings_small.csv')
+    return credits_df, keywords_df, links_df, movies_df, ratings_df
+
+
+def draw_adult_movies_pie_chart(movies_df):
+    plt.figure(figsize=(8, 4))
+    plt.scatter(x=[0.5, 1.5], y=[1, 1], s=15000, color=['#06837f', '#fdc100'])
+    plt.xlim(0, 2)
+    plt.ylim(0.9, 1.2)
+
+    plt.title('Distribution of Adult and Non Adult Movies', fontsize=18, weight=600, color='#333d29')
+    plt.text(0.5, 1, '{}\nMovies'.format(str(len(movies_df[movies_df['adult'] == 'True']))), va='center', ha='center',
+             fontsize=18, weight=600, color='white')
+    plt.text(1.5, 1, '{}\nMovies'.format(str(len(movies_df[movies_df['adult'] == 'False']))), va='center', ha='center',
+             fontsize=18, weight=600, color='white')
+    plt.text(0.5, 1.11, 'Adult', va='center', ha='center', fontsize=17, weight=500, color='#1c2541')
+    plt.text(1.5, 1.11, 'Non Adult', va='center', ha='center', fontsize=17, weight=500, color='#1c2541')
+
+    plt.axis('off')
+
+    plt.savefig('adult.png')
+    mlflow.log_artifact('adult.png')
+
+
+def draw_genres_pie_chart(df):
+    genres_list = []
+    for i in df['genres']:
+        i = i[1:]
+        i = i[:-1]
+        genres_list.extend(i.split(', '))
+
+    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(14, 6))
+
+    df_plot = pd.DataFrame(Counter(genres_list).most_common(5), columns=['genre', 'total'])
+    # ax = sns.barplot(data=df_plot, x='genre', y='total', ax=axes[0],
+    #                  palette=['#06837f', '#02cecb', '#b4ffff', '#f8e16c', '#fed811'])
+    # ax.set_title('Top 5 Genres in Movies', fontsize=18, weight=600, color='#333d29')
+    # sns.despine()
+
+    df_plot_full = pd.DataFrame([Counter(genres_list)]).transpose().sort_values(by=0, ascending=False)
+    df_plot.loc[len(df_plot)] = {'genre': 'Others', 'total': df_plot_full[6:].sum()[0]}
+    plt.title('Percentage Ratio of Movie Genres', fontsize=18, weight=600, color='#333d29')
+    wedges, texts, autotexts = axes[1].pie(x=df_plot['total'], labels=df_plot['genre'], autopct='%.2f%%',
+                                           textprops=dict(fontsize=14), explode=[0, 0, 0, 0, 0, 0.1],
+                                           colors=['#06837f', '#02cecb', '#b4ffff', '#f8e16c', '#fed811', '#fdc100'])
+
+    for autotext in autotexts:
+        autotext.set_color('#1c2541')
+        autotext.set_weight('bold')
+
+    axes[1].axis('off')
+
+    plt.savefig('genres.png')
+    mlflow.log_artifact('genres.png')
+
+
+def director(x):
+    for i in x:
+        if i["job"] == "Director":
+            return i["name"]
+    return ""
+
+
+def writer_screenplay(x):
+    names = []
+    for i in x:
+        if (i["job"] == "Writer") | (i["job"] == "Screenplay") | (i["job"] == "Author"):
+            name = i["name"]
+            names.append(name)
+    return names
+
+
+def calculate_cosine_similarity(train_df):
+    cosine_sim = cosine_similarity(train_df)
+    return cosine_sim
+
+
+def clean_data(credits_df, keywords_df, movies_df):
+    # draw_adult_movies_pie_chart(movies_df)
+    # Cast id column to int
+    movies_df["id"] = movies_df["id"].apply(pd.to_numeric, errors="ignore")
+    keywords_df["id"] = keywords_df["id"].apply(int)
+    credits_df["id"] = credits_df["id"].apply(int)
+
+    # Merge movies, keywords, credits based on id column
+    df = movies_df.merge(keywords_df, on="id").merge(credits_df, on="id")
+
+    """Cleaning our merged data from from duplicated and null values"""
+
+    # Find null values in our merged data frame
+    df.isnull().sum()
+
+    # Remove duplicated values with the same titles
+    df.drop_duplicates(subset=["title", "id"], inplace=True)
+
+    # Remove movies with null titles
+    df = df[df.title.notnull()]
+
+    # Find number of movies with vote count < 30
+    (df.vote_count < 30).sum()
+
+    # Remove movies with vote count < 30
+    df = df[df.vote_count > 30]
+
+    # Make release data numeric
+    df["release_date"] = pd.to_datetime(df['release_date'])
+    df["release_year"] = df["release_date"].dt.year
+
+    df.drop("release_date", axis=1, inplace=True)
+
+    # Remove null values
+    df = df[df["release_year"].notnull()]
+    df = df[df["runtime"].notnull()]
+
+    # Make vote_average and release_year column categorical and normalize them
+    df["vote_average_bins"] = pd.cut(df["vote_average"].astype(float), 10, labels=range(1, 11))
+    scaler = MinMaxScaler()
+    df["vote_average_bins"] = df["vote_average_bins"].astype(int)
+    df["vote_average_bins"] = scaler.fit_transform(df["vote_average_bins"].values.reshape(-1, 1))
+
+    df["release_year_bins"] = pd.qcut(df["release_year"].astype(float), q=10, labels=range(1, 11))
+    scaler = MinMaxScaler()
+    df["release_year_bins"] = df["release_year_bins"].astype(int)
+    df["release_year_bins"] = scaler.fit_transform(df["release_year_bins"].values.reshape(-1, 1))
+
+    # Set data frame primary index to title
+    df.set_index("title", inplace=True)
+
+    # Make languages one-hotted
+    languages = pd.get_dummies(df["original_language"])
+
+    # Extract genre name from json
+    df['genres'] = df['genres'].fillna('[]').apply(literal_eval).apply(
+        lambda x: [i['name'] for i in x] if isinstance(x, list) else "")
+    df["genres"] = df["genres"].astype(str)
+    # draw_genres_pie_chart(df)
+
+    # Make genres one-hotted
+    cv = CountVectorizer(lowercase=False)
+    genres = cv.fit_transform(df["genres"])
+    genres_df = pd.DataFrame(genres.todense(), columns=cv.get_feature_names_out())
+    genres_df.set_index(df.index, inplace=True)
+
+    # Make keywords,tagline,overview one-hotted
+    df['keywords'] = df['keywords'].fillna('[]').apply(literal_eval).apply(
+        lambda x: [i['name'] for i in x] if isinstance(x, list) else "")
+    df["keywords"] = df["keywords"].astype(str)
+    df["tagline"].fillna("", inplace=True)
+    df["overview"].fillna("", inplace=True)
+    df["keywords"].fillna("", inplace=True)
+    df["text"] = df["overview"] + df["tagline"] + df["keywords"]
+
+    tfidf = TfidfVectorizer(max_features=5000)
+    tfidf_matrix = tfidf.fit_transform(df["text"])
+    tfidf_df = pd.DataFrame(tfidf_matrix.todense(), columns=tfidf.get_feature_names_out())
+    tfidf_df.set_index(df.index, inplace=True)
+
+    # Make cast one-hotted
+    df['cast'] = df['cast'].fillna('[]').apply(literal_eval).apply(
+        lambda x: [i['name'] for i in x] if isinstance(x, list) else "")
+    df["cast"] = df["cast"].apply(lambda x: [c.replace(" ", "") for c in x])
+    df["cast"] = df["cast"].apply(lambda x: x[:15])
+    df["CC"] = df["cast"].astype(str)
+    cv = CountVectorizer(lowercase=False, min_df=4)
+    cast = cv.fit_transform(df["CC"])
+    cast_df = pd.DataFrame(cast.todense(), columns=cv.get_feature_names_out())
+    cast_df.set_index(df.index, inplace=True)
+
+    df["dir"] = df["crew"].apply(literal_eval).apply(director)
+    directors = pd.get_dummies(df["dir"])
+
+    df["writer_screenplay"] = df["crew"].apply(literal_eval).apply(writer_screenplay)
+    df["writer_screenplay"] = df["writer_screenplay"].apply(lambda x: [c.replace(" ", "") for c in x])
+    df["writer_screenplay"] = df["writer_screenplay"].apply(lambda x: x[:3])
+    df["writer_screenplay"] = df["writer_screenplay"].astype(str)
+    cv = CountVectorizer(lowercase=False, min_df=2)
+    writing = cv.fit_transform(df["writer_screenplay"])
+    writing_df = pd.DataFrame(writing.todense(), columns=cv.get_feature_names_out())
+    writing_df.set_index(df.index, inplace=True)
+
+    gc.collect()
+    train_df = pd.concat([languages, genres_df, cast_df, writing_df, directors, tfidf_df], axis=1)
+    train_df = train_df.astype(np.int8)
+    gc.collect()
+
+    return train_df, df
+
+
+class RecommenderSystem(mlflow.pyfunc.PythonModel):
+    def load_context(self, context):
+        credits_df, keywords_df, links_df, movies_df, ratings_df = load_data()
+        self.train_df, self.df = clean_data(credits_df, keywords_df, movies_df)
+        self.cosine_sim = calculate_cosine_similarity(self.train_df)
+
+    def predict(self, context, model_input):
+        return self.recommend(model_input[0], self.cosine_sim)
+
+    def recommend(self, title, cosine_sim):
+        indices = pd.Series(range(0, len(self.train_df.index)), index=self.train_df.index).drop_duplicates()
+        number = 10
+        # Get the index of the movie that matches the title
+        idx = indices[title]
+        # Get the pairwsie similarity scores of all movies with that movie
+        sim_scores = list(enumerate(cosine_sim[idx]))
+
+        # Sort the movies based on the similarity scores
+        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
+
+        scores_arr = np.array(sim_scores)
+        scores_mean = np.average(scores_arr, axis=0)
+        mlflow.log_metric("cosine-total-avg", scores_mean[1])
+
+        # Get the scores of the 10 most similar movies
+        sim_scores = sim_scores[1:number + 1]
+        scores_arr = np.array(sim_scores)
+        scores_mean = np.average(scores_arr, axis=0)
+
+        mlflow.log_metric("cosine-result-avg", scores_mean[1])
+        mlflow.log_metric("cosine-result-max", sim_scores[0][1])
+        mlflow.log_metric("cosine-result-min", sim_scores[number - 1][1])
+        mlflow.log_param("number-of-results", number)
+
+        # Get the movie indices
+        movie_indices = [i[0] for i in sim_scores]
+
+        recommendations = pd.DataFrame({"Movies": self.df.iloc[movie_indices].index.tolist(),
+                                        "Id": self.df.iloc[movie_indices].imdb_id.tolist(),
+                                        "Similarity": [sim[1] for sim in sim_scores]})
+        return recommendations
+
+
+if __name__ == '__main__':
+    mlflow.pyfunc.save_model(path="imdb-recommendation-v2", python_model=RecommenderSystem())
+    init_mlflow()
+    mlflow.pyfunc.log_model("imdb-recommendation-v2", python_model=RecommenderSystem(), registered_model_name="recommendation-model-v2")
+    loaded_model = mlflow.pyfunc.load_model("imdb-recommendation-v2")
+    print(loaded_model.predict(["The Dark Knight Rises"]))
+    mlflow.end_run()
+