Integrate backend

app.py

@@ -1,65 +1,213 @@
 import gradio as gr
+import pandas as pd
+import os
+from scripts.genbit_metrics import *
+from scripts.gender_profession_tagging import *
+from scripts.gender_tagging import *
+from utils.load_csv import *
+from utils.read_config import get_args
 
-def run_evaluation(dataset_id, methodology):
-    return f'Running evaluation for {dataset_id} with {methodology}'
+dummy_data = [
+    ["Category", "Value", "Percentage"],
+    ["Total Reviews", 50000, None],
+    ["Total Sentences", 621647, None],
+    ["Pronouns in Sentences", None, None],
+    ["Male Pronouns", 85615, None],
+    ["Female Pronouns", 39372, None],
+    ["Both Male and Female Pronouns", 7765, None],
+    ["Exclusive Usage of Pronouns", None, None],
+    ["Only Male Pronouns", 77860, 13.77],
+    ["Only Female Pronouns", 31617, 6.33],
+    ["Pronouns and Professions in Sentences", None, None],
+    ["Male Pronouns with Professions", 5580, 0.9],
+    ["Female Pronouns with Professions", 2618, 0.42],
+    ["Exclusive Usage of Pronouns with Professions", None, None],
+    ["Only Male Pronouns with Professions", 5011, 0.81],
+    ["Only Female Pronouns with Professions", 2049, 0.33],
+    ["Pronouns and Professions in Combination", None, None],
+    ["Male or Female Pronouns with Professions", 7629, 1.23],
+    ["Male and Female Pronouns with Professions", 569, 0.09]
+]
 
-    if methodology == 'A':
-        run_a(dataset_id)
-    elif methodology == 'B':
-        run_b(dataset_id)
-    elif methodology == 'C':
-        run_c(dataset_id)
+
+def display_methodology(methodology):
+    title = "### " + methodology
+    description = ""
+
+    if methodology == "Term Identity Diversity Analysis":
+        description = "333"
+    elif methodology == "Gender Label Evaluation":
+        description = "This approach to addressing gender bias in language places a strong emphasis on a fundamental shift in detection and mitigation strategies.\n- Instead of solely relying on traditional frequency-based methods, this approach adopts a more nuanced perspective, prioritizing features within the text that consider contextual and semantic cues. It recognizes that gender bias extends beyond mere word frequency and delves into how language is structured and how it reinforces gender stereotypes.\n- Even with advanced models like Word Embedding and Contextual Word Embedding, which capture more complex language features, there's still a risk of inheriting biases from training data.\n- To tackle this, this approach advocates for a data-driven strategy, involving the collection and labeling of datasets encompassing various subtypes of bias, using a comprehensive taxonomy for precise categorization."
+    elif methodology == "Microsoft Genbit":
+        description = "GenBiT is a versatile tool designed to address gender bias in language datasets by utilizing word co-occurrence statistical methods to measure bias. It introduces a novel approach to mitigating gender bias by combining contextual data augmentation, random sampling, sentence classification, and targeted gendered data filtering.\n- The primary goal is to reduce historical gender biases within conversational parallel multilingual datasets, ultimately enhancing the fairness and inclusiveness of machine learning model training and its subsequent applications.\n- What sets GenBiT apart is its adaptability to various forms of bias, not limited to gender alone. It can effectively address biases related to race, religion, or other dimensions, making it a valuable generic tool for bias mitigation in language datasets.\n- GenBiT's impact extends beyond bias reduction metrics; it has shown positive results in improving the performance of machine learning classifiers like Support Vector Machine(SVM). Augmented datasets produced by GenBiT yield significant enhancements in f1-score when compared to the original datasets, underlining its practical benefits in machine learning applications."
+
+    return (
+        gr.Markdown.update(title, visible=True),
+        gr.Markdown.update(description, visible=True)
+    )
+
+
+def run_evaluation(dataset_file, dataset_scope, dataset_scope_n, dataset_columns, methodology):
+
+    status = {}
+    # Read CSV file
+    dataset = check_csv(dataset_file.name)
+    sample_method = dataset_scope
+    col_name = dataset_columns
+    num_sample_records = dataset_scope_n
+
+    # Check selected methodology
+    if methodology == "Term Identity Diversity Analysis":
+        status = load_dataset_and_analyze_gender_tag(dataset, sample_method, col_name, num_sample_records)
+    if methodology == "Gender Label Evaluation":
+        status = load_dataset_and_analyze_gender_profession(dataset, sample_method, col_name, num_sample_records)
+    if methodology == "Microsoft Genbit":
+        status = load_dataset_and_get_genbit_metrics(dataset, sample_method, col_name, num_sample_records)
     
-def schema_uploaded_file(file):
-    file_paths = [file.name for file in file]
-    return file_paths
+    # status = {
+    #     "dataset": dataset_file.name,
+    #     "methodology": methodology,
+    #     "scope": dataset_scope + " " + str(dataset_scope_n),
+    #     "column": dataset_columns
+    # }
 
-demo = gr.Blocks(theme=gr.themes.Soft())
+    return gr.JSON.update(status, visible=True)
 
-with demo:
-    gr.Markdown("# BiasAware: Dataset Bias Detection")
+    if methodology == "A":
+        run_a(dataset)
+    elif methodology == "B":
+        run_b(dataset)
+    elif methodology == "C":
+        run_c(dataset)
+
+
+def process_dataset(dataset):
     
+    data = pd.read_csv(dataset.name)
+    # maximum_records = get_args("first_records")
+    # input_records = data.shape(0)
+    # num_sample_records = input_records if input_records < maximum_records else maximum_records
+
+    columns = data.columns.tolist()
+    columns = [x for x in columns if data[x].dtype == "object"]
+
+    return (
+        gr.Radio.update(
+            label="Scope",
+            info="Determines the scope of the dataset to be analyzed",
+            choices=["First", "Last", "Random"],
+            value="First",
+            visible=True,
+            interactive=True,
+        ),
+        gr.Slider.update(
+            label="Number of Entries",
+            info=f"Determines the number of entries to be analyzed. The dataset has {data.shape[0]} entries.",
+            minimum=1,
+            maximum=data.shape[0],
+            value=data.shape[0] // 2,
+            visible=True,
+            interactive=True
+        ),
+        gr.Radio.update(
+            label="Column",
+            info="Determines the column to be analyzed. These are the columns with text data.",
+            choices=columns,
+            value=columns[0],
+            visible=True,
+            interactive=True,
+        )
+    )
+
+
+def process_column(dataset, column):
+    data = pd.read_csv(dataset.name)
+    corpus = data[column].to_list()[:10]
+
+    return gr.Dataframe.update(value=pd.DataFrame({"Data Corpus": corpus}), max_rows=5, visible=True)
+
+
+BiasAware = gr.Blocks(title="BiasAware: Dataset Bias Detection")
+
+with BiasAware:
+    gr.Markdown("# BiasAware: Dataset Bias Detection")
+    gr.Markdown(
+        "Natural Language Processing (NLP) training datasets often reflect the biases present in the data sources they are compiled from, leading to the **perpetuation of stereotypes, underrepresentation, and skewed perspectives in AI models**. BiasAware is designed to **identify and quantify biases present in text data**, making it an invaluable resource for data scientists, machine learning practitioners, and organizations committed to **mitigating bias in AI systems**."
+    )
+
     with gr.Row():
         with gr.Column(scale=1):
-            gr.Markdown("Select a dataset to analyze")
+            gr.Markdown("## Dataset")
+
+            dataset_file = gr.File()
+            dataset_examples = gr.Examples(
+                [
+                    os.path.join(os.path.dirname(__file__), "data/z_animal.csv"),
+                    os.path.join(os.path.dirname(__file__), "data/z_employee.csv"),
+                    os.path.join(os.path.dirname(__file__), "data/z_house.csv"),
+
+                ],
+                inputs=dataset_file,
+            )
+
+            dataset_scope = gr.Radio(visible=False)
+            dataset_scope_n = gr.Slider(visible=False)
+            dataset_columns = gr.Radio(visible=False)
+
+            dataset_corpus = gr.Dataframe(visible=False)
+
+        with gr.Column(scale=1):
+            gr.Markdown("## Methodology")
 
-            dataset_id = gr.Text(label="Dataset")
-            gr.Examples(
-                examples=["imdb", "amazon_reviews_multi", "tweet_eval"],
-                fn=run_evaluation,
-                inputs=[dataset_id]
+            methodology = gr.Radio(
+                label="Methodology",
+                info="Determines the methodology to be used for bias detection",
+                choices=[
+                    "Term Identity Diversity Analysis",
+                    "Gender Label Evaluation",
+                    "Microsoft Genbit",
+                ],
+                value="Term Identity Diversity Analysis",
             )
 
-            methodology = gr.Dropdown(["Term Identity Diversity Analysis", "Textual Gender Label Evaluation", "GenBit"], label="Methodology")
+            evalButton = gr.Button("Run Evaluation")
 
-            button = gr.Button("Run Evaluation")
+            methodology_title = gr.Markdown(visible=False)
+            methodology_description = gr.Markdown(visible=False)
 
         with gr.Column(scale=4):
-            gr.Markdown("### Results")
+            gr.Markdown("## Result")
 
-            with gr.Box():
-                methodology_title = gr.Markdown("### Identity Term Sampling")
-                methodology_description = gr.Markdown("lorem ipsum")
-            
-            methodology_test_description = gr.Markdown("lorem ipsum")
-            outputs = gr.Markdown()
-            gr.Error("No results to display")
-        
+            result_status = gr.JSON(visible=False)
+            result = gr.DataFrame()
+
+    dataset_file.change(
+        fn=process_dataset,
+        inputs=[dataset_file],
+        outputs=[
+            dataset_scope,
+            dataset_scope_n,
+            dataset_columns
+        ]
+    )
+
+    dataset_columns.change(
+        fn=process_column,
+        inputs=[dataset_file, dataset_columns],
+        outputs=[dataset_corpus],
+    )
 
     methodology.change(
-        fn=lambda x: (f'### {x}', "lorem ipseum", "lorem ipsum"),
+        fn=display_methodology,
         inputs=[methodology],
-        outputs=[methodology_title, methodology_description, methodology_test_description]
+        outputs=[methodology_title, methodology_description],
     )
 
-    button.click(
+    evalButton.click(
         fn=run_evaluation,
-        inputs=[dataset_id, methodology],
-        outputs=[outputs]
+        inputs=[dataset_file, dataset_scope,
+                dataset_scope_n, dataset_columns, methodology],
+        outputs=[result_status]
     )
 
-     # For user dataset upload
-    gr.Interface(schema_uploaded_file, "file", "text")
-
-
-demo.launch()
+BiasAware.launch()

scripts/genbit_metrics.py

@@ -1,6 +1,7 @@
 from genbit.genbit_metrics import GenBitMetrics
 import pandas as pd
 from utils.read_config import get_args
+from utils.load_csv import load_sample
 
 
 def cal_metrics(dataset):
@@ -19,37 +20,24 @@ def cal_metrics(dataset):
 
   return metrics
 
-# Function to load sample of dataset
-def load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name):
-    
-    # Keep only requireed column
-    df = df[[col_name]]
-    if sample_method == "first_record" and df.shape[0] > sample_first_records:
-        df = df.iloc[:sample_first_records].copy().reset_index()
-    if sample_method == "random_pick" and df.shape[0] > sample_first_records:
-        df = df.sample(sample_first_records, random_state=sample_random_seed).copy().reset_index()
-    return df
-
 
 # Function to extract genbit metrics
 def extract_genbit_metris(stats):
     metrics = {}
-    metrics["genbit_score"] = stats["genbit_score"]
-    metrics["percentage_of_female_gender_definition_words"] = stats["percentage_of_female_gender_definition_words"]
-    metrics["percentage_of_male_gender_definition_words"] = stats["percentage_of_male_gender_definition_words"]
-    metrics["percentage_of_non_binary_gender_definition_words"] = stats["percentage_of_non_binary_gender_definition_words"]
-    metrics["percentage_of_trans_gender_definition_words"] = stats["percentage_of_trans_gender_definition_words"]
-    metrics["percentage_of_cis_gender_definition_words"] = stats["percentage_of_cis_gender_definition_words"]
-    metrics["num_words_considered"] = stats["statistics"]["num_words_considered"]
+    metrics["genbit_score"] = str(stats["genbit_score"])
+    metrics["percentage_of_female_gender_definition_words"] = str(stats["percentage_of_female_gender_definition_words"])
+    metrics["percentage_of_male_gender_definition_words"] = str(stats["percentage_of_male_gender_definition_words"])
+    metrics["percentage_of_non_binary_gender_definition_words"] = str(stats["percentage_of_non_binary_gender_definition_words"])
+    metrics["percentage_of_trans_gender_definition_words"] = str(stats["percentage_of_trans_gender_definition_words"])
+    metrics["percentage_of_cis_gender_definition_words"] = str(stats["percentage_of_cis_gender_definition_words"])
+    metrics["num_words_considered"] = str(stats["statistics"]["num_words_considered"])
 
     return metrics
 
-def load_dataset_and_get_genbit_metrics(df, sample_method, col_name):
-    # Get args from config file
-    sample_first_records = get_args("first_records")
-    sample_random_seed = get_args("random_seed")
+def load_dataset_and_get_genbit_metrics(df, sample_method, col_name, num_sample_records):
+ 
 
-    sample_df = load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name)
+    sample_df = load_sample(num_sample_records, sample_method, df, col_name)
 
     # Turn into a list of text.
     sample_text = sample_df[col_name].tolist()

scripts/gender_profession_tagging.py

@@ -9,6 +9,7 @@ import multiprocessing.pool
 import warnings
 warnings.filterwarnings("ignore")
 from utils.read_config import get_args
+from utils.load_csv import load_sample
 
 
 # For sentence split
@@ -88,12 +89,12 @@ def get_statistics(results_df):
     count_female_pronoun_profession = results_df[(results_df["Female Pronoun"] != "") & (results_df["Profession"] != "")]["Female Pronoun"].count()
 
     return{
-        "total_sentence" : count_total_sentence,
-        "both_gender_prof_match" : count_both_match,
-        "count_male_pronoun" : count_male_pronoun,
-        "count_female_pronoun" : count_female_pronoun,
-        "count_male_pronoun_profession" : count_male_pronoun_profession,
-        "count_female_pronoun_profession" : count_female_pronoun_profession
+        "total_sentence" : str(count_total_sentence),
+        "both_gender_prof_match" : str(count_both_match),
+        "count_male_pronoun" : str(count_male_pronoun),
+        "count_female_pronoun" : str(count_female_pronoun),
+        "count_male_pronoun_profession" : str(count_male_pronoun_profession),
+        "count_female_pronoun_profession" : str(count_female_pronoun_profession)
     }
 
 # Function to return regular expression patterns
@@ -109,23 +110,11 @@ def get_regex_pattern(male_pronoun, female_pronoun, professions):
     
     return male_pronoun_pat, female_pronoun_pat, professions_pat
 
-# Function to load sample of dataset
-def load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name):
-    
-    # Keep only requireed column
-    df = df[[col_name]]
-    if sample_method == "first_record" and df.shape[0] > sample_first_records:
-        df = df.iloc[:sample_first_records].copy().reset_index()
-    if sample_method == "random_pick" and df.shape[0] > sample_first_records:
-        df = df.sample(sample_first_records, random_state=sample_random_seed).copy().reset_index()
-    return df
-
-def load_dataset_and_analyze_gender_profession(df, sample_method, col_name):
+
+def load_dataset_and_analyze_gender_profession(df, sample_method, col_name, num_sample_records):
     # Get args from config file
-    sample_first_records = get_args("first_records")
-    sample_random_seed = get_args("random_seed")
 
-    sample_df = load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name)
+    sample_df = load_sample(num_sample_records, sample_method, df, col_name)
 
 
     # Lowercase of text

scripts/gender_tagging.py

@@ -2,6 +2,7 @@
 import pandas as pd
 import re
 from utils.read_config import get_args
+from utils.load_csv import load_sample
 
 # Function to get count of male terms in text
 def count_male_terms(text, male_terms):
@@ -43,17 +44,6 @@ def get_gender_tag(count_m_term, count_f_term):
     return tag
 
 
-# Function to load sample of dataset
-def load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name):
-    
-    # Keep only requireed column
-    df = df[[col_name]]
-    if sample_method == "first_record" and df.shape[0] > sample_first_records:
-        df = df.iloc[:sample_first_records].copy().reset_index()
-    if sample_method == "random_pick" and df.shape[0] > sample_first_records:
-        df = df.sample(sample_first_records, random_state=sample_random_seed).copy().reset_index()
-    return df
-
 # Function to calculate PG and SPG
 def get_pg_spg(sample_df):
     count_no_gender_sentences = sample_df[sample_df["gender_cat"] == "No Gender"]['gender_cat'].count()
@@ -68,28 +58,22 @@ def get_pg_spg(sample_df):
     count_female_spg = sample_df[sample_df['gender_cat'] == "Female Stronly Positive Gender"]['gender_cat'].count()
     
     return {
-    "gender" : count_gender_sentences,
-    "no gender" : count_no_gender_sentences,
-    "equal gender" : count_equal_gender,
-    "female pg" : count_female_pg,
-    "male pg" : count_male_pg,
-    "female spg" : count_female_spg,
-    "male spg" : count_male_spg
+    "gender" : str(count_gender_sentences),
+    "no gender" : str(count_no_gender_sentences),
+    "equal gender" : str(count_equal_gender),
+    "female pg" : str(count_female_pg),
+    "male pg" : str(count_male_pg),
+    "female spg" : str(count_female_spg),
+    "male spg" : str(count_male_spg)
     }
 
 # Function to load dataset and get the analysis done
-def load_dataset_and_analyze_gender_tag(df, sample_method, col_name):
+def load_dataset_and_analyze_gender_tag(df, sample_method, col_name, num_sample_records):
     # Read config file
-
     male_terms = get_args("male_terms")
     female_terms = get_args("female_terms")
-    sample_first_records = get_args("first_records")
-    sample_random_seed = get_args("random_seed")
-
-    
-
-    sample_df = load_sample(sample_first_records, sample_random_seed, sample_method, df, col_name)
-
+    # Load sample
+    sample_df = load_sample(num_sample_records, sample_method, df, col_name)
 
     # Lowercase of text
     sample_df[col_name] = sample_df[col_name].str.lower().str.strip()

utils/config.json

@@ -1,5 +1,5 @@
 {
-"first_records" : 200,
+"first_records" : 2000,
 "random_seed" : 42,
 "male_terms" : ["man", "boy", "male", "he", "son", "his", "himself", "guy", "father", "john"],
 "female_terms" : ["woman", "girl", "female", "she", "daughter", "her", "herself", "gal", "mother", "mary"],