correct mismatch column names unique df

bangalore_score.py

@@ -126,9 +126,9 @@ class Bangalore_Score(evaluate.Metric):
 
             # parse headers
             if len(rows) > 0:
-                headers_row = rows[0].strip().strip('|')                    # remove trailing pipes
+                headers_row = rows[0].strip().strip('|')                        # remove trailing pipes
                 headers_list = [x.strip() for x in headers_row.split('|')]      # split by pipes and remove whitespace
-                headers_text = ' '.join(sorted(headers_list))               # join headers
+                headers_text = ' '.join(sorted(headers_list))                   # join headers
 
             # try parse records
             if len(rows) > 2:
@@ -145,7 +145,7 @@ class Bangalore_Score(evaluate.Metric):
                         pass
 
                 # normalize rows (set column order with sorted headers)
-                sorted_headers = sorted(headers_list)
+                sorted_headers = sorted(set(list(headers_list)))
                 df = pd.DataFrame(records, columns=sorted_headers)  # create dataframe
                 df = df.sort_values(by=sorted_headers)            # sort rows
                 # csv
@@ -166,22 +166,21 @@ class Bangalore_Score(evaluate.Metric):
                 'num_rows': [len(records)],
             }
 
-        ### run computations ###
+        ### normalize ###
         pred_ds = Dataset.from_dict({'text': predictions})
         refs_ds = Dataset.from_dict({'text': references})
         proc_ds = DatasetDict({'predictions': pred_ds, 'references': refs_ds})
         proc_ds = proc_ds.map(normalize_fn, num_proc=num_proc)
 
+
+        ### compute metrics ###
+        # https://huggingface.co/evaluate-metric
+
         # exact_match (headers)
         exact_match_headers = evaluate.load('exact_match').compute(
             predictions=proc_ds['predictions']['headers_text'], references=proc_ds['references']['headers_text']
         )['exact_match']
 
-        # charcut (data)
-        # charcut = evaluate.load('BramVanroy/CharCut').compute(
-        #     predictions=proc_ds['predictions']['rows_text'], references=proc_ds['references']['rows_text']
-        # )['charcut_mt']
-
         # bleu (data)
         bleu = evaluate.load('bleu').compute(
             predictions=proc_ds['predictions']['rows_text'], references=proc_ds['references']['rows_text']
@@ -197,7 +196,6 @@ class Bangalore_Score(evaluate.Metric):
             predictions=proc_ds['predictions']['rows_text'], references=proc_ds['references']['rows_text']
         )['levenshtein_distance']
 
-
         # row accuracy (num rows)
         # row_accuracy = evaluate.load('accuracy').compute(
         #     predictions=proc_ds['predictions']['num_rows'], references=proc_ds['references']['num_rows']