import gradio as gr import pandas as pd import os from apscheduler.schedulers.background import BackgroundScheduler from huggingface_hub import HfApi from uploads import add_new_eval CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r"""@inproceedings{iltur-2024, title = "IL-TUR: Benchmark for Indian Legal Text Understanding and Reasoning", author = "Joshi, Abhinav and Paul, Shaunak Sharma, Akshat and Goyal, Pawan and Ghosh, Saptarshi and Modi, Ashutosh", booktitle = "Proceedings of the 62st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", month = aug, year = "2024", address = "Bangkok, Thailand", publisher = "Association for Computational Linguistics", } }""" api = HfApi() TOKEN = os.environ.get("TOKEN", None) LEADERBOARD_PATH = f"Exploration-lab/IL-TUR-Leaderboard" def restart_space(): api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN) # Function to load data from a given CSV file def baseline_load_data(tasks): # version = version.replace("%", "p") file_path = f"submissions/baseline/baseline.csv" # Replace with your file paths df = pd.read_csv(file_path) # we only want specific columns and in a specific order column_names = [ "Method", "Submitted By", "L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM", "Average", ] if tasks is None: breakpoint() # based on the tasks, remove the columns that are not needed if "L-NER" not in tasks: column_names.remove("L-NER") if "RR" not in tasks: column_names.remove("RR") if "CJPE" not in tasks: column_names.remove("CJPE") if "BAIL" not in tasks: column_names.remove("BAIL") if "LSI" not in tasks: column_names.remove("LSI") if "PCR" not in tasks: column_names.remove("PCR") if "SUMM" not in tasks: column_names.remove("SUMM") df = df[column_names] df = df.sort_values(by="Average", ascending=False) df = df.drop_duplicates(subset=["Method"], keep="first") return df def load_data(tasks): baseline_df = baseline_load_data(tasks) return baseline_df # Function for searching in the leaderboard def search_leaderboard(df, query): if query == "": return df else: return df[df["Method"].str.contains(query)] # Function to change the version of the leaderboard def change_version(tasks): new_df = load_data(tasks) return new_df # Initialize Gradio app demo = gr.Blocks() with demo: gr.Markdown( """ ## 🥇 IL-TUR Leaderboard Legal systems worldwide are inundated with exponential growth in cases and documents. There is an imminent need to develop NLP and ML techniques for automatically processing and understanding legal documents to streamline the legal system. However, evaluating and comparing various NLP models designed specifically for the legal domain is challenging. This paper addresses this challenge by proposing IL-TUR: Benchmark for Indian Legal Text Understanding and Reasoning. IL-TUR contains monolingual (English, Hindi) and multi-lingual (9 Indian languages) domain-specific tasks that address different aspects of the legal system from the point of view of understanding and reasoning over Indian legal documents. We present baseline models (including LLM-based) for each task, outlining the gap between models and the ground truth. We will release a public leaderboard where the research community can upload and compare legal text understanding systems on various metrics, thus fostering research in the legal domain. Read more at [https://exploration-lab.github.io/IL-TUR/](https://exploration-lab.github.io/IL-TUR/). """ ) with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, elem_id="citation-button", show_copy_button=True, ) # .style(show_copy_button=True) with gr.Tabs(): with gr.TabItem("Leaderboard"): with gr.Row(): tasks_checkbox = gr.CheckboxGroup( label="Select Tasks", choices=["L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM"], value=["L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM"], ) with gr.Row(): search_bar = gr.Textbox( placeholder="Search for methods...", show_label=False, ) leaderboard_table = gr.components.Dataframe( value=load_data( # "baseline", ["L-NER", "RR", "CJPE", "BAIL", "LSI", "PCR", "SUMM"], ), interactive=True, visible=True, ) # version_dropdown.change( # change_version, # inputs=[model_dropdown, version_dropdown, tasks_checkbox], # outputs=leaderboard_table, # ) # model_dropdown.change( # change_version, # inputs=[model_dropdown, version_dropdown, tasks_checkbox], # outputs=leaderboard_table, # ) search_bar.change( search_leaderboard, inputs=[ leaderboard_table, search_bar, # tasks_checkbox ], outputs=leaderboard_table, ) tasks_checkbox.change( change_version, inputs=[tasks_checkbox], outputs=leaderboard_table, ) with gr.Accordion("Submit a new model for evaluation"): with gr.Row(): with gr.Column(): method_name_textbox = gr.Textbox(label="Method name") url_textbox = gr.Textbox(label="Url to model information") with gr.Column(): organisation = gr.Textbox(label="Organisation") mail = gr.Textbox(label="Contact email") file_output = gr.File() submit_button = gr.Button("Submit Eval") submission_result = gr.Markdown() submit_button.click( add_new_eval, [ method_name_textbox, url_textbox, file_output, organisation, mail, ], submission_result, ) gr.Markdown( """ ## Quick Links - [**Website**](https://exploration-lab.github.io/IL-TUR): The landing page for IL-TUR - [**arXiv Paper**](https://arxiv.org/abs/2307.05260): Detailed information about the IL-TUR dataset and its significance in unlearning tasks. - [**GitHub Repository**](https://github.com/exploration-lab/IL-TUR): Access the source code, fine-tuning scripts, and additional resources for the IL-TUR dataset. - [**Dataset on Hugging Face**](https://huggingface.co/datasets/Exploration-Lab/IL-TUR): Direct link to download the IL-TUR dataset. - [**Leaderboard on Hugging Face Spaces**](https://huggingface.co/spaces/Exploration-Lab/IL-TUR_leaderboard): Current rankings and submissions for the IL-TUR dataset challenges. ## Loading the Dataset To load the dataset, use the following code: ```python from datasets import load_dataset dataset = load_dataset("Exploration-Lab/IL-TUR","") ``` """ ) # scheduler = BackgroundScheduler() # scheduler.add_job(restart_space, "interval", seconds=1800) # scheduler.start() # demo.queue(default_concurrency_limit=40).launch() # demo.launch() scheduler = BackgroundScheduler() scheduler.add_job(restart_space, "interval", seconds=3600) scheduler.start() demo.launch(debug=True)