app.py

import gradio as gr
from pyvis.network import Network
import networkx as nx
import numpy as np
import pandas as pd
import os
from datasets import load_dataset
from datasets import Features
from datasets import Value
from datasets import Dataset
import matplotlib.pyplot as plt
import re

pattern = r'"(.*?)"'
# this pattern captures anything in a double quotes. 

Secret_token = os.getenv('HF_token')

dataset = load_dataset('FDSRashid/hadith_info',data_files = 'Basic_Edge_Information.csv', token = Secret_token, split = 'train')

edge_info = dataset.to_pandas()

features = Features({'Rawi ID': Value('int32'), 'Famous Name': Value('string'), 'Narrator Rank': Value('string'), 'Number of Narrations': Value('string'),  'Generation': Value('string')})
narrator_bios = load_dataset("FDSRashid/hadith_info", data_files = 'Teacher_Bios.csv', token = Secret_token,features=features )
narrator_bios = narrator_bios['train'].to_pandas()
narrator_bios.loc[49845, 'Narrator Rank'] = 'رسول الله'
narrator_bios.loc[49845, 'Number of Narrations'] = 0
narrator_bios['Number of Narrations'] = narrator_bios['Number of Narrations'].astype(int)
narrator_bios.loc[49845, 'Number of Narrations'] = 327512
# 8125 Narrators have no Generation, listed in dataset as None
narrator_bios['Generation'] = narrator_bios['Generation'].replace([None], [-1])
narrator_bios['Generation'] = narrator_bios['Generation'].astype(int)

features = Features({'matn': Value('string'), 'taraf_ID': Value('string'), 'bookid_hadithid': Value('string')})

dataset = load_dataset("FDSRashid/hadith_info", data_files = 'All_Matns.csv',token = Secret_token, features = features)
matn_info = dataset['train'].to_pandas()
matn_info = matn_info.drop(97550)
matn_info = matn_info.drop(307206)
matn_info['taraf_ID'] = matn_info['taraf_ID'].replace('KeyAbsent', -1)

matn_info['taraf_ID'] = matn_info['taraf_ID'].astype(int)

# Isnad Info Hadiths column is structured like {"BookNum_HadithNum", ...} for each edge
isnad_info = load_dataset('FDSRashid/hadith_info',token = Secret_token, data_files = 'isnad_info.csv', split = 'train').to_pandas()
isnad_info['Hadiths Cleaned'] = isnad_info['Hadiths'].apply(lambda x: [re.findall(pattern, string)[0].split("_") for string in x[1:-1].split(',')])
# Hadiths Cleaned is a list of lists, each sub-list is Book Id, Hadith ID
taraf_max = np.max(matn_info['taraf_ID'].unique())
isnad_info['Tarafs Cleaned'] = isnad_info['Tarafs'].apply(lambda x: np.array([int(i.strip(' ')) for i in x[1:-1].split(',')]))

cmap = plt.colormaps['cool']

books = load_dataset('FDSRashid/Hadith_info', data_files='Books.csv', token = Secret_token)['train'].to_pandas()

matn_info['Book_ID'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[0]))
matn_info['Hadith Number'] = matn_info['bookid_hadithid'].apply(lambda x: int(x.split('_')[1]))
matn_info = pd.merge(matn_info, books, on='Book_ID')

def value_to_hex(value):
    rgba_color = cmap(value)
    return "#{:02X}{:02X}{:02X}".format(int(rgba_color[0] * 255), int(rgba_color[1] * 255), int(rgba_color[2] * 255))


def visualize_subTaraf(taraf_num, hadith_str, yaxis):
    hadith_list = hadith_str.split(',')
    hadith_list = [hadith.strip() for hadith in hadith_list]
    hadiths = np.array([], dtype=int)
    for hadith in hadith_list:
        if '-' in hadith:
            if hadith.count('-') > 1:
                #print('Please use only one Dash mark!')
                raise gr.Error('Please use only one Dash mark!')
            hadith_multi = hadith.strip().split('-')
            if any([not had.isnumeric() for had in hadith_multi]):
                #print('Invalid Begining')
                raise gr.Error('Invalid Begining')
            elif len(hadith_multi) != 2:
                #print('Two numbers for a range of Hadith numbers please!')
                raise gr.Error('Two numbers for a range of Hadith numbers please!')
            hadith_multi = [int(had) for had in hadith_multi]
            hadiths = np.append(hadiths, np.arange(hadith_multi[0], hadith_multi[1]))
        elif hadith.isnumeric():
            hadiths = np.append(hadiths, int(hadith))
        else:
            #print('Invalid Data format!')
            raise gr.Error("Invalid Data format!")
    return hadiths
    

def visualize_isnad(taraf_num, yaxis):
    taraf = matn_info[matn_info['taraf_ID'] == taraf_num]
    taraf_hadith = taraf['bookid_hadithid'].to_list()
    taraf_matns = taraf['matn'].to_list()
    taraf_hadith_split = [i.split('_') for i in taraf_hadith]
    taraf_book = taraf['Book_Name'].to_list()
    taraf_author = taraf['Author'].to_list()
    taraf_hadith_number = taraf['Hadith Number'].to_list()
    lst_hadith = []
    hadith_cleaned = isnad_info['Tarafs Cleaned'].apply(lambda x: taraf_num in x)
    isnad_hadith = isnad_info[hadith_cleaned]
    for i in range(len(taraf_hadith_split)):
        # This checks each hadith in the Taraf, is that book id hadith id found in each of the edges of isnad_info
        #This loop get the end transmitter of each Hadith in the Taraf
        isnad_in_hadith1 = isnad_hadith['Hadiths Cleaned'].apply(lambda x: taraf_hadith_split[i] in x )
        isnad_hadith1 = isnad_hadith[isnad_in_hadith1][['Source', 'Destination']]
        G = nx.from_pandas_edgelist(isnad_hadith1, source = 'Source', target = 'Destination', create_using = nx.DiGraph())
        node = [int(n) for n, d in G.out_degree() if d == 0]
        for n in node:
            gen_node = narrator_bios[narrator_bios['Rawi ID']==n]['Generation'].to_list()
            if len(gen_node):
                gen_node = gen_node[0]
            else:
                gen_node = -1
            name_node = narrator_bios[narrator_bios['Rawi ID']==n]['Famous Name'].to_list()
            if len(name_node):
                name_node = name_node[0]
            else:
                name_node = 'فلان'
            lst_hadith.append([taraf_matns[i], gen_node, name_node, taraf_book[i], taraf_author[i], taraf_hadith_number[i], str(n), str(i)])
    df = pd.DataFrame(lst_hadith, columns = ['Matn', 'Generation', 'Name', 'Book_Name', 'Author', 'Book Hadith Number', 'End Transmitter ID', 'Hadith Number'])
    
    isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list())
    isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list())
    isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان')
    isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان')

    end_nodes = df['End Transmitter ID'].tolist() 
    G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph())
    isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot')
    x_stretch = 4
    y_stretch = 4
    net = Network(directed =True)

    for node, pos in isnad_pos.items():
        node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)]
        student_narrations = node_info['Number of Narrations'].to_list()
        if len(student_narrations):
            student_narrations = student_narrations[0]
        else:
            student_narrations = 1
        student_gen = node_info['Generation'].to_list()
        if len(student_gen):
            student_gen = student_gen[0]
        else:
            student_gen = -1
        student_rank = node_info["Narrator Rank"].to_list()
        if len(student_rank):
            student_rank = student_rank[0]
        else:
            student_rank = 'فلان'
        node_name = node_info['Famous Name'].to_list()
        if len(node_name):
            node_name = node_name[0]
        else:
            node_name = 'فلان'
        if node == '99999':
            net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70)   
        elif node in end_nodes:
            end_matn_info = df[df["End Transmitter ID"] == node]
            net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen} \n Hadith {" ".join(end_matn_info["Hadith Number"].tolist())}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50)
        else:
            net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50)
    for _, row in isnad_hadith.iterrows():
        source = row['Source']
        target = row['Destination']
        net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}")
    net.toggle_physics(False)    
    html = net.generate_html()
    html = html.replace("'", "\"")
    df = df.rename(columns = {'Generation': 'Gen.', 'Book Hadith Number': 'Hdth Num', 'End Transmitter ID': 'End Narrator ID', 'Hadith Number': 'Index', 'Book_Name': 'Book', 'Name':'Final Narrator'})
    return f"""<iframe style="width: 100%; height: 600px;margin:0 auto" name="result" allow="midi; geolocation; microphone; camera; 
  display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
  allow-scripts allow-same-origin allow-popups 
  allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
  allowpaymentrequest="" frameborder="0" srcdoc='{html}'></iframe>""" , df.drop('Hdth Num', axis=1)
    

def taraf_booknum(taraf_num):
    taraf = matn_info[matn_info['taraf_ID'] == taraf_num]
    return taraf[['matn', 'Book_ID', 'Hadith Number', 'Book_Name', 'Author']]

def visualize_hadith_isnad(df, yaxis):
    df['bookid_hadithid'] = df['Book_ID'].astype(str) + '_' + df['Hadith Number'].astype(str)
    hadith = matn_info[matn_info['bookid_hadithid'].isin(df['bookid_hadithid'])]
    taraf_hadith_split = [i.split('_') for i in hadith['bookid_hadithid'].to_list()]
    hadith_cleaned = isnad_info['Hadiths Cleaned'].apply(lambda x: any(i in x for i in taraf_hadith_split))
    isnad_hadith = isnad_info[hadith_cleaned]
    isnad_hadith['Teacher'] = isnad_hadith['Source'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list())
    isnad_hadith['Student'] = isnad_hadith['Destination'].apply(lambda x: narrator_bios[narrator_bios['Rawi ID'].astype(int) == int(x)]['Famous Name'].to_list())
    isnad_hadith['Teacher'] = isnad_hadith['Teacher'].apply(lambda x: x[0] if len(x)==1 else 'فلان')
    isnad_hadith['Student'] = isnad_hadith['Student'].apply(lambda x: x[0] if len(x)==1 else 'فلان')

    G = nx.from_pandas_edgelist(isnad_hadith, source = 'Source', target = 'Destination', create_using = nx.DiGraph())
    isnad_pos = nx.nx_agraph.graphviz_layout(G, prog='dot')
    x_stretch = 4
    y_stretch = 4
    net = Network(directed =True)

    for node, pos in isnad_pos.items():
        node_info = narrator_bios[narrator_bios['Rawi ID'] == int(node)]
        student_narrations = node_info['Number of Narrations'].to_list()
        if len(student_narrations):
            student_narrations = student_narrations[0]
        else:
            student_narrations = 1
        student_gen = node_info['Generation'].to_list()
        if len(student_gen):
            student_gen = student_gen[0]
        else:
            student_gen = -1
        student_rank = node_info["Narrator Rank"].to_list()
        if len(student_rank):
            student_rank = student_rank[0]
        else:
            student_rank = 'فلان'
        node_name = node_info['Famous Name'].to_list()
        if len(node_name):
            node_name = node_name[0]
        else:
            node_name = 'فلان'
        if node == '99999':
            net.add_node(node, font = {'size':50, 'color': 'black'}, color = '#000000', label = f'{node_name} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 70)   
        # elif node in end_nodes:
        #     end_matn_info = df[df["End Transmitter ID"] == node]
        #     net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen} \n Hadith {" ".join(end_matn_info["Hadith Number"].tolist())}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50)
        else:
            net.add_node(node, font = {'size':30, 'color': 'red'}, color = value_to_hex(student_narrations), label = f'{node_name} \n {student_rank} \n ID: {node} - Gen {student_gen}', x= pos[0]*x_stretch, y= -1*pos[1]*y_stretch, size= 50)
    for _, row in isnad_hadith.iterrows():
        source = row['Source']
        target = row['Destination']
        net.add_edge(source, target, color = value_to_hex(int(row[f'{yaxis} Count'])), label = f"{row[f'{yaxis} Count']}")
    net.toggle_physics(False)    
    html = net.generate_html()
    html = html.replace("'", "\"")
    return f"""<iframe style="width: 100%; height: 600px;margin:0 auto" name="result" allow="midi; geolocation; microphone; camera; 
  display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
  allow-scripts allow-same-origin allow-popups 
  allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
  allowpaymentrequest="" frameborder="0" srcdoc='{html}'></iframe>"""  


with gr.Blocks() as demo:
    with gr.Tab("Whole Taraf Visualizer"):
        Yaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.')  
        taraf_number = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1)
        btn = gr.Button('Submit')
        btn.click(fn = visualize_isnad, inputs = [taraf_number, Yaxis], outputs = [gr.HTML(), gr.DataFrame(wrap=True, column_widths=[43, 8, 11,11,10,8, 9])])
    with gr.Tab("Book and Hadith Number Retriever"):
        taraf_num = gr.Slider(1,taraf_max , value=10000, label="Taraf", info="Choose the Taraf to Input", step = 1)
        btn_num = gr.Button('Retrieve')
        btn_num.click(fn=taraf_booknum, inputs = [taraf_num], outputs= [gr.DataFrame(wrap=True)])
    with gr.Tab('Select Hadith Isnad Visualizer'):
        yyaxis = gr.Dropdown(choices = ['Taraf', 'Hadith', 'Isnad', 'Book'], value = 'Taraf', label = 'Variable to Display', info = 'Choose the variable to visualize.')
        hadith_selection =  gr.Dataframe(
            headers=["Book_ID", "Hadith Number"],
            datatype=["number", "number"],
            row_count=5,
            col_count=(2, "fixed"))
        btn_hadith = gr.Button('Visualize')
        btn_hadith.click(fn=visualize_hadith_isnad, inputs=[hadith_selection, yyaxis], outputs=[gr.HTML()])
demo.launch()