first commit

.gitignore

@@ -0,0 +1,9 @@
+
+secrets.toml
+__pycache__
+
+# *.txt
+*.tsv
+*.csv
+*.json
+

README.md

@@ -1,7 +1,7 @@
 ---
 title: De Identification Leaderboard
-emoji: 📊
-colorFrom: purple
+emoji: 🏃
+colorFrom: pink
 colorTo: yellow
 sdk: streamlit
 sdk_version: 1.35.0
@@ -11,3 +11,33 @@ license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+
+
+# de-identification-leaderboard
+
+##  leaderboard data
+score wil save to huggingface dataset
+[zhaorui-nb/leaderboard-score](https://huggingface.co/datasets/zhaorui-nb/leaderboard-score)
+
+
+## submit 
+###  filename format 
+replace '/' to '@'
+```
+[{Organization@Model}][{Dataaset}][{Method}]{Filename}.txt"
+```
+
+### line in answer txt (tsv)
+```
+{file_name}\t{label_type}\t{label_start}\t{label_end}\t{label_text}\n
+```
+
+## Support dataset
+```
+Setting1
+Setting2
+Setting3
+```
+

app.py

@@ -0,0 +1,224 @@
+import streamlit as st
+import pandas as pd
+import os
+from utils.Evaluation_answer_txt import Evaluation_answer_txt
+from utils.upload_hub import upload_scores_to_hub, file_name_decode
+import time
+import json
+import datasets
+from huggingface_hub import HfApi
+from huggingface_hub import hf_hub_download
+# st.set_page_config(layout="wide")
+st.set_page_config(layout="centered")
+st.markdown(
+    f"""
+    <style>
+        .appview-container .main .block-container{{
+            max-width: 80%;
+            padding: 50px;
+        }}
+    </style>
+    """,
+    unsafe_allow_html=True
+)
+
+@st.cache_data
+def download_gold_answer(repo, filename, token, force_download=False):
+    ret = hf_hub_download(repo_id=repo, repo_type='dataset',  filename=filename, token=token, force_download=force_download)
+    return ret
+
+
+HUB_TOKEN = st.secrets['hf']
+HUB_API = HfApi(token=HUB_TOKEN)
+
+LEADERBOARD_DATASET_REPO = 'zhaorui-nb/leaderboard-score'
+# Setting1 Setting2 Setting3
+
+ANSWER_REPO = 'zhaorui-nb/leaderboard-answer'
+GET_GOLD_ANSWER_PATH = {
+    'Setting1': download_gold_answer(ANSWER_REPO, 'dataset/Setting1_test_answer.txt', HUB_TOKEN),
+    'Setting2': download_gold_answer(ANSWER_REPO, 'dataset/Setting2_test_answer.txt', HUB_TOKEN),
+    'Setting3': download_gold_answer(ANSWER_REPO, 'dataset/Setting3_test_answer.txt', HUB_TOKEN)
+}
+
+
+# cache the dataset in the session state
+def get_leaderboard_df():
+    with st.spinner('Loading leaderboard data...'):
+        if st.session_state.get('leaderboard_df') is None:
+            dataset = datasets.load_dataset(LEADERBOARD_DATASET_REPO)  
+            df = pd.DataFrame(dataset['train'])
+            st.session_state['leaderboard_df'] = df
+            return df
+        else:
+            return st.session_state['leaderboard_df']
+
+
+st.title('De-identification Model Leaderboard')
+
+try:
+    with st.container():
+        # columns
+        # ['model name', 'dataset', 'method', 'file name', 'submitter',
+        #    'MICRO precision', 'MICRO recall', 'MICRO f1', 'MACRO precision',
+        #    'MACRO recall', 'MACRO f1', 'detail result']
+
+        df = get_leaderboard_df()
+        # replace model name column @ to /
+        df['model name'] = df['model name'].str.replace('@', '/')
+
+        # remove the detail result column
+        default_columns = [c for c in df.columns if c not in ['detail result']]
+        selected_columns = st.multiselect('Select columns to display', df.columns, default=default_columns)
+
+        leaderboard_df =    st.dataframe(df[selected_columns], selection_mode='multi-row', on_select='rerun', key='leaderboard')
+        
+        st.subheader("Detail Result")
+        det_ind = st.session_state.leaderboard['selection']['rows']
+        if len(det_ind) == 0:
+            st.write(f'Please check the boxes to view the detailed results.')
+        else:
+            col_detial = st.columns(len(det_ind))
+            for i, dind in enumerate(det_ind):
+                with col_detial[i]:
+                    dis = f"{df.iloc[dind]['model name']}___{df.iloc[dind]['dataset']}___{df.iloc[dind]['method']}"
+                    color = [st.success, st.info, st.warning, st.error]
+                    color[i % 4](dis)
+                    
+                    dic = df.iloc[dind]['detail result']
+                    dt_df = pd.DataFrame(dic).T
+                    st.dataframe(dt_df)
+
+except Exception as e:
+    st.error(f"Error: {e}")
+
+st.markdown("---")
+
+# ############################################################################################################
+# ############################################### Evaluation_answer_txt
+# ############################################################################################################
+
+model_name_input = ''
+dataset_input = ''
+method_input = ''
+file_name = ''
+submitter_input = ''
+
+if 'score_json' not in st.session_state:
+    st.session_state['score_json'] = None
+
+@st.cache_data()
+def get_file_info(uploaded_file):
+    filename_info = file_name_decode(uploaded_file.name)
+    return filename_info
+
+@st.cache_data()
+def eval_answer_txt(set_name, uploaded_file):
+    print(f"eval_answer_txt: {time.time()}" , set_name)
+    
+    if set_name not in GET_GOLD_ANSWER_PATH:
+        return None
+    gold_answer_txt = GET_GOLD_ANSWER_PATH[set_name]
+    eval = Evaluation_answer_txt(gold_answer_txt, uploaded_file)
+    score_json = eval.eval()
+    return score_json
+
+def clear_score_json():
+    st.session_state['score_json'] = None
+
+st.title("Model Evaluation")
+st.write("Support file naming: [{Organization@Model}][{Dataaset}][{Method}]{Filename}.txt")
+
+col_upload = st.columns([3,1])
+with col_upload[0]:
+    uploaded_file = st.file_uploader("Please upload the answer.txt file", type=["txt"], key="uploaded_file", on_change=clear_score_json)
+with col_upload[1]:
+    if not uploaded_file:
+        st.warning("please upload file")
+        st.session_state['score_json'] = None
+    else:
+        st.success("file uploaded successfully")
+        
+        filename_info = get_file_info(uploaded_file)
+        if filename_info:
+            model_name_input = filename_info['model_name']
+            dataset_input = filename_info['dataset']
+            method_input = filename_info['method']
+            file_name = filename_info['file_name']
+
+col_score = st.columns([7,5])
+if uploaded_file:
+    with col_score[1], st.container(border=True):
+        model_name_input = st.text_input("model name", model_name_input)
+        dataset_input = st.text_input("dataset", dataset_input)
+        method_input = st.text_input("method", method_input)
+        file_name = st.text_input("file name", file_name)
+        submitter_input = st.text_input("submitter", submitter_input)
+        check_all_fill_in = model_name_input and dataset_input and method_input and file_name and submitter_input
+        
+        col_sumit_and_recalculate = st.columns(2)
+        with col_sumit_and_recalculate[0]:
+            calculate_btn = st.button("calculate", type='secondary', use_container_width=True)
+        with col_sumit_and_recalculate[1]:
+            submit_btn = st.button("SUBMIT", type='primary', use_container_width=True , disabled=not check_all_fill_in)
+    
+    if calculate_btn or st.session_state['score_json'] is None:
+        set_name = dataset_input
+        st.session_state['score_json'] = eval_answer_txt(set_name, uploaded_file)
+        if st.session_state['score_json']:
+            st.success("evaluation success")
+        else:
+            st.error("evaluation failed, please check the file content or set the correct dataset name.")
+
+if st.session_state['score_json']:
+    with col_score[0], st.container(border=True):
+        df = pd.DataFrame(st.session_state['score_json']).T
+        # split the column MICRO_AVERAGE and MACRO_AVERAGE into another dataframe
+        tag_df = df.drop(["MICRO_AVERAGE", "MACRO_AVERAGE"], axis=0)
+        avg_df = df.loc[["MICRO_AVERAGE", "MACRO_AVERAGE"]]
+
+        col_sort_func = st.columns(2)
+        
+        with col_sort_func[0]:
+            sorted_column = st.selectbox("选择排序列", df.columns)
+
+        with col_sort_func[1]:
+            ascending = st.radio("Sort Order", ["Ascending", "Descending"])
+
+        tag_df = tag_df.sort_values(by=sorted_column, ascending=ascending=="Ascending")
+
+        st.dataframe(pd.concat([tag_df, avg_df]), use_container_width=True)
+
+
+    if not check_all_fill_in:
+        st.warning("Please fill in the complete information.")
+
+    if submit_btn:
+        if st.session_state['score_json']:
+            score_json = st.session_state['score_json']
+            
+            leaderboard_dict = {
+                "model name": model_name_input,
+                "dataset": dataset_input,
+                "method": method_input,
+                "file name": file_name,
+                "submitter": submitter_input,
+
+                "MICRO precision": score_json["MICRO_AVERAGE"]["precision"],
+                "MICRO recall": score_json["MICRO_AVERAGE"]["recall"],
+                "MICRO f1": score_json["MICRO_AVERAGE"]["f1"],
+                "MACRO precision": score_json["MACRO_AVERAGE"]["precision"],
+                "MACRO recall": score_json["MACRO_AVERAGE"]["recall"],
+                "MACRO f1": score_json["MACRO_AVERAGE"]["f1"],
+                "detail result": score_json
+            }
+
+            repo_file_path = f'data/train-[{model_name_input}][{dataset_input}][{method_input}][{file_name}].json'
+            upload_res = upload_scores_to_hub(HUB_API, leaderboard_dict, repo_file_path, hub_repo=LEADERBOARD_DATASET_REPO)
+            if upload_res:
+                st.success(f"submit success")
+                st.success(f"your score at here: {upload_res}")
+            else:
+                st.error("submit failed")
+
+

utils/Evaluation_answer_txt.py

@@ -0,0 +1,180 @@
+import re 
+import os 
+from collections import Counter
+import json
+
+
+class Tag:
+    def __init__(self, txt_line:str):
+        # | file_name | label_type | label_start | label_end | label_text |
+        # match = re.match(r'(.+)\t(\w+)\t(\d+)\t(\d+)\t(.+)', txt_line)
+        try:
+            sep = txt_line.strip().split('\t')
+            self.file_id = sep[0]    
+            self.type = sep[1]
+            self.start = sep[2] # int(sep[2])
+            self.end =  sep[3] # int(sep[3])
+            self.text = sep[4]
+        except:
+            raise ValueError('The format of the input line is not correct. Please check the input line format.')
+
+    def get_type(self):
+        return self.type
+
+    def get_file_id(self):
+        return self.file_id
+    
+    def __eq__(self, other: 'Tag'):
+        # if all file_id, type, start, end,  are the same, return True
+        # text is not considered for the comparison
+        ck_file_id = self.file_id == other.file_id
+        ck_type = self.type == other.type
+        ck_start = self.start == other.start
+        ck_end = self.end == other.end
+        # ck_text = self.text == other.text
+        if ck_file_id and ck_type and ck_start and ck_end:
+            return True
+        else:
+            return False
+    def __repr__(self):
+        return f'<{self.__class__.__name__} {self.file_id:10} {self.type:10} s:{self.start:5} e:{self.end:5} {self.text}>\n'
+
+    def __hash__(self):
+        return hash((self.file_id, self.type, self.start, self.end))
+    
+class Evaluation_answer_txt:
+    def __init__(self, gold_answer, pred_answer):
+        self.gold_answer = gold_answer
+        self.pred_answer = pred_answer
+
+        self.gold_set = set() # set of Tag
+        self.pred_set = set() # set of Tag
+
+        self.type_set = set() # set of label type str
+        self.gold_label_counter = Counter() # Counter of gold label type
+
+        self.resault_score = {}
+
+    def _lines_to_tag_set(self, lines, set_type): # set_type: 'gold' or 'pred'
+        tags = []
+        for i in range(len(lines)):
+            try:
+                tag = Tag(lines[i])
+                tags.append(tag)
+            except:
+                print(f'Error at {set_type} answer line: {i+1}, {lines[i]}')
+        return set(tags)
+    
+    def _set_filter(self, tag_set, type):
+        # tag set filter by type
+        return {tag for tag in tag_set if tag.get_type() == type}
+    
+    def _division(self, a, b):
+        try:
+            return a / b
+        except:
+            return 0.0
+
+    def _f1_score(self, TP=None, FP=None, FN=None):
+        if TP is None or FP is None or FN is None:
+            raise ValueError('TP, FP, FN should be given.')
+
+        precision = self._division(TP, TP + FP)
+        recall = self._division(TP, TP + FN)
+        f1 = self._division(2 * precision * recall, precision + recall)
+            
+        return {'precision': precision, 'recall': recall, 'f1': f1}            
+        
+
+    def eval(self, ignore_no_gold_tag_file=True):
+        with open(self.gold_answer, 'r') as f:
+            gold_line = f.readlines()
+        # with open(self.pred_answer, 'r') as f:
+        #     pred_line = f.readlines()
+        ########## add to support the input is a file object ##########
+        if isinstance(self.pred_answer, str):
+            with open(self.pred_answer, 'r') as f:
+                pred_line = f.readlines()
+
+
+        else:
+            pred_line = self.pred_answer.readlines()
+            #pred_line is bytes, need to decode
+            pred_line = [line.decode('utf-8') for line in pred_line]
+
+        self.gold_set = self._lines_to_tag_set(gold_line, 'gold')
+        self.pred_set = self._lines_to_tag_set(pred_line, 'pred')
+
+        # in islab aicup program, it will ignore the files that have no gold tags
+        # that program only consider the files that write in gold answer.txt
+        if ignore_no_gold_tag_file:
+            # filter the files that have no gold tags
+            gold_files = {tag.get_file_id() for tag in self.gold_set}
+            self.pred_set = {tag for tag in self.pred_set if tag.get_file_id() in gold_files}
+
+        # statistics tags and types 
+        for tag in self.gold_set:
+            self.type_set.add(tag.get_type())
+            self.gold_label_counter[tag.get_type()] += 1
+        for tag in self.pred_set:
+            self.type_set.add(tag.get_type())
+
+        TP_set = self.gold_set & self.pred_set
+        FP_set = self.pred_set - self.gold_set
+        FN_set = self.gold_set - self.pred_set
+
+        # count each type of label
+        for label in self.type_set:
+            filter_TP = self._set_filter(TP_set, label)
+            filter_FP = self._set_filter(FP_set, label)
+            filter_FN = self._set_filter(FN_set, label)
+            score = self._f1_score(len(filter_TP), len(filter_FP), len(filter_FN))
+            self.resault_score[label] = score
+       
+        # MICRO_AVERAGE
+        self.resault_score['MICRO_AVERAGE'] = self._f1_score(len(TP_set), len(FP_set), len(FN_set))
+
+        # MACRO_AVERAGE
+        precision_sum = 0
+        recall_sum = 0
+        # f1_sum = 0 # at aicup, calc by MACRO_AVERAGE precision and recall
+        for label in self.type_set:
+            precision_sum += self.resault_score[label]['precision']
+            recall_sum += self.resault_score[label]['recall']
+            # f1_sum += self.resault_score[label]['f1']
+        
+        precision = self._division(precision_sum, len(self.type_set))
+        recall = self._division(recall_sum, len(self.type_set))
+        # f1 = 2 * precision * recall / (precision + recall)
+        f1 = self._division(2 * precision * recall , (precision + recall))
+
+        self.resault_score['MACRO_AVERAGE'] = {'precision': precision, 'recall': recall, 'f1': f1}
+
+        # add Support to each type of label
+        for label in self.type_set:
+            self.resault_score[label]['support'] = self.gold_label_counter[label]
+        self.resault_score['MICRO_AVERAGE']['support'] = len(self.gold_set)
+        self.resault_score['MACRO_AVERAGE']['support'] = len(self.gold_set)
+
+        # return json.dumps(self.resault_score, indent=4)
+        return self.resault_score
+
+
+if __name__=="__main__":
+    # with open('.output/[meta-llama@Llama-2-7b-hf][Setting3][icl]answer.txt', 'r', encoding='utf-8') as f:
+    #     lines = [line.strip() for line in f.readlines() if line.strip() != '']
+
+    # gold_path = 'dataset/Setting3_test_answer.txt'
+    # pred_path = '.output/EleutherAI-pythia-1b-Setting3_answer.txt'
+
+
+    # gold_path = './.output/test_eval/gold_answer.txt'
+    # pred_path = './.output/test_eval/pred_answer.txt'
+
+    gold_path = 'dataset/Setting3_test_answer.txt'
+    pred_path = '.output/[meta-llama@Llama-2-7b-hf][Setting3][icl]answer.txt'
+
+
+    eval = Evaluation_answer_txt(gold_path, pred_path)
+    res = eval.eval()
+    print(res)

utils/upload_hub.py

@@ -0,0 +1,56 @@
+import json 
+import uuid
+import os 
+import re
+from huggingface_hub import HfApi
+from huggingface_hub import hf_hub_download
+
+
+
+def file_name_decode(file_name):
+    # model_name,dataset,method,answer.txt
+    # input file name example: [[email protected]][Setting3][icl]answer.txt
+    
+    match = re.match(rf'\[([^\[^\]]+)\]\[([^\[^\]]+)\]\[([^\[^\]]+)\]([^\[^\]]+)', file_name)
+    
+    if match:
+        model_name, dataset, method, file_name = match.groups()
+        ret_dict = {
+            'model_name': model_name,
+            'dataset': dataset,
+            'method': method,
+            'file_name': file_name
+        }
+        return ret_dict
+    return None    
+
+def upload_scores_to_hub(api, scores_dict, path_in_repo,hub_repo='zhaorui-nb/test_json'):
+    # id = str(uuid.uuid4())
+    save_json_path = f'.output/upload.json'   
+    os.makedirs(os.path.dirname(save_json_path), exist_ok=True)
+    with open(save_json_path, 'w') as f:
+        json.dump(scores_dict, f , indent=4)
+
+    # SAVE JSON TO HUB 
+    res = api.upload_file(
+        path_or_fileobj=save_json_path,
+        path_in_repo=path_in_repo, #f'data/train,{os.path.basename(save_json_path)}',
+        repo_id=hub_repo,
+        repo_type="dataset",
+    )
+
+    return res
+
+
+
+
+
+if __name__ == "__main__":
+
+    pass
+
+
+
+
+
+