Hugging Face's logo

Spaces:
Lihuchen
/
pearl_leaderboard
Sleeping

App Files Community
pearl_leaderboard
File size: 5,032 Bytes 
3f0817b
 
 
 
 
 
 
80be486
0821caa
3f0817b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
04d8cb5
3f0817b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 

__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']

import gradio as gr
import pandas as pd



COLUMN_NAMES = ["Model", "Size", "Avg",  "PPDB", "PPDB filtered", "Turney", "BIRD", "YAGO", "UMLS", "CoNLL", "BC5CDR", "AutoFJ"]

UNTUNED_MODEL_RESULTS = '''[FastText](https://fasttext.cc/)                    &--&94.4&61.2&59.6&58.9&16.9&14.5&3.0&0.2&53.6 \\
[Sentence-BERT](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2)      &110M&94.6&66.8&50.4&62.6&21.6&23.6&25.5&48.4&57.2 \\
[Phrase-BERT](https://huggingface.co/whaleloops/phrase-bert)      &110M&96.8&68.7&57.2&68.8&23.7&26.1&35.4&59.5&66.9 \\
[UCTopic](https://github.com/JiachengLi1995/UCTopic)      &240M&91.2&64.6&60.2&60.2&5.2&6.9&18.3&33.3&29.5 \\
[E5-small](https://huggingface.co/intfloat/e5-small-v2)      &34M&96.0&56.8&55.9&63.1&43.3&42.0&27.6&53.7&74.8 \\
[E5-base](https://huggingface.co/intfloat/e5-base-v2)      &110M&95.4&65.6&59.4&66.3&47.3&44.0&32.0&69.3&76.1\\
[PEARL-small](https://huggingface.co/Lihuchen/pearl_small)      &34M& 97.0&70.2&57.9&68.1& 48.1&44.5&42.4&59.3&75.2\\
[PEARL-base](https://huggingface.co/Lihuchen/pearl_base)      &110M&97.3&72.2&59.7&72.6&50.7&45.8&39.3&69.4&77.1\\'''


def parse_line(line):
    model_results = line.replace(" ", "").strip("\\").split("&")
    for i in range(1, len(model_results)):
        if i == 1:
            res = model_results[1]
        else:
            res = float(model_results[i])
        model_results[i] = res
    return model_results
    
def get_baseline_df():
    df_data = []
    
    lines = UNTUNED_MODEL_RESULTS.split("\n")
    for line in lines:
        model_results = parse_line(line)
        print(model_results)
        assert len(model_results) == 11
        avg = sum(model_results[2:]) / 9
        model_results.insert(2, avg)
        #model_results.insert(1, "False")
        df_data.append(model_results)
    # lines = TUNED_MODEL_RESULTS.split("\n")
    # for line in lines:
    #     model_results = parse_line(line)
    #     assert len(model_results) == 10
    #     avg = sum(model_results[1:-3] + model_results[-2:]) / 8
    #     model_results.insert(1, avg)
    #     model_results.insert(1, "True")
    #     df_data.append(model_results)
        
    print(len(df_data))
    df = pd.DataFrame(df_data, columns=COLUMN_NAMES).round(1)
    print(df.head())
    return df


CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""@article{chen2024learning,
  title={Learning High-Quality and General-Purpose Phrase Representations},
  author={Chen, Lihu and Varoquaux, Ga{\"e}l and Suchanek, Fabian M},
  journal={arXiv preprint arXiv:2401.10407},
  year={2024}
}
}"""


block = gr.Blocks()

with block:
    gr.Markdown(
        """# 🦪⚪ The PEARL-Leaderboard aims to evaluate string embeddings on various tasks.
    🏆 Our PEARL leaderboard contains 9 phrase-level datasets of five types of tasks, covering both the tasks of data science and natural language processing. <br>
    | **[ 📜 paper](https://arxiv.org/pdf/2401.10407.pdf)** |  **[🤗 PEARL-small](https://huggingface.co/Lihuchen/pearl_small)** |  **[🤗 PEARL-base](https://huggingface.co/Lihuchen/pearl_base)** | 🤗 **[PEARL-Benchmark](https://huggingface.co/datasets/Lihuchen/pearl_benchmark)** |
    **[💾 data](https://zenodo.org/records/10676475)** |
    """
    )
    
    gr.Markdown(
        """ ## Task Description<br>
* **Paraphrase Classification**: PPDB and PPDBfiltered ([Wang et al., 2021](https://aclanthology.org/2021.emnlp-main.846/))
* **Phrase Similarity**: Turney ([Turney, 2012](https://arxiv.org/pdf/1309.4035.pdf)) and BIRD ([Asaadi et al., 2019](https://aclanthology.org/N19-1050/))
* **Entity Retrieval**: We constructed two datasets based on Yago ([Pellissier Tanon et al., 2020](https://hal-lara.archives-ouvertes.fr/DIG/hal-03108570v1)) and UMLS ([Bodenreider, 2004](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC308795/))
* **Entity Clustering**: CoNLL 03 ([Tjong Kim Sang, 2002](https://aclanthology.org/W02-2024/)) and BC5CDR ([Li et al., 2016](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4860626/))
* **Fuzzy Join**: AutoFJ benchmark ([Li et al., 2021](https://arxiv.org/abs/2103.04489)) contains 50 diverse fuzzy-join datasets 

    """
    )
    with gr.Row():
        data = gr.components.Dataframe(
            type="pandas", datatype=["markdown", "markdown", "number", "number", "number", "number", "number", "number", "number", "number", "number", "number"]
        )
    with gr.Row():
        data_run = gr.Button("Refresh")
        data_run.click(
            get_baseline_df, outputs=data
        )

    with gr.Row():
        with gr.Accordion("Citation", open=True):
            citation_button = gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                elem_id="citation-button",
            )
            #.style(show_copy_button=True)
    block.load(get_baseline_df, outputs=data)

block.launch()