save

bangalore_score.py

@@ -105,19 +105,36 @@ class Bangalore_Score(evaluate.Metric):
         # evaluate.load('ncoop57/levenshtein_distance')
         pass
 
+    # todo: compartmentalize... and soften
+    #  * extract_records(text, return_df=False, omit_columns=[])
+    #  * preprocess_fn(example), omit, extract, ...
+
     def normalize_fn(
             self,
             example,
             text_field='text',
             unk_token='Not Disclosed',
+            omit_columns=True,
             return_df=False
     ):
         """
-        parse output text into headers, rows, and records
-        - parse row by row (incomplete rows)
+        Normalize Markdown Text String to rtext or DataFrame
+            * fill NaNs with unk_token
+            * assumes markdown table format
+            * assumes headers are present
+            * assumes headers are unique
+            * does not handle special tokens
+        Normalization:
+            * columns   ->      pd.DataFrame(x, columns=sorted(x.columns))
+            * rows      ->      pd.DataFrame.sort_values(by=sorted(x.columns))
+            * replaces ->      NaNs with unk_token
+            * omit      ->      columns with text > max_chars
         :param example:
+            {
+                'text': <markdown table>,
+                'omit_columns': <list of columns to omit>
+            }
         :return:
-        Note: this does not handle special tokens
         expected input format:
 
         | col1 | col2 | col3 |      <- start and trailing pipes required
@@ -129,7 +146,10 @@ class Bangalore_Score(evaluate.Metric):
 
         records = []
         rows_text = unk_token
-        omit_columns = example.get('omit_columns', [])
+        if omit_columns:
+            omit_columns = example.get('omit_columns', [])
+        else:
+            omit_columns = []
 
         text = dict(example)[text_field]
         text = text.strip()
@@ -186,6 +206,8 @@ class Bangalore_Score(evaluate.Metric):
             csv_norm = df.to_csv(index=False, sep='|')
             csv_norm  = csv_norm.replace('|', ' | ')    # add spaces around pipes
             csv_norm = csv_norm.replace('\r', '')       # remove carriage returns
+            csv_norm = csv_norm.replace('\t', ' ')      # remove tabs
+
             # only rows text
             rows_text = csv_norm.split('\n')[1:-1]
             rows_text = '\n'.join(rows_text).strip()
@@ -244,7 +266,12 @@ class Bangalore_Score(evaluate.Metric):
         """
         scores_list = []
         for ref, pred in zip([reference]*len(predictions), predictions):
-            score_dict = metric.compute(references=[ref], predictions=[pred])
+            score_dict = metric.compute(
+                references=[ref],
+                predictions=[pred],
+                # ignore_case=True,
+                # ignore_punctuation=True,
+            )
             if isinstance(score_dict, dict):
                 score = score_dict.get(metric_key, 0)
             elif isinstance(score_dict, float):
@@ -316,6 +343,7 @@ class Bangalore_Score(evaluate.Metric):
             metric,
             metric_key,
             best='max',
+            omit_columns=True
     ):
         """Returns the scores"""
         import json
@@ -332,8 +360,8 @@ class Bangalore_Score(evaluate.Metric):
         proc_ds = proc_ds.map(self.ref_omit_columns, desc='omit_columns (ref)')
 
         # 3. normalize predictions and references
-        predictions_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='pred'))
-        references_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='ref'))
+        predictions_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='pred', omit_columns=omit_columns))
+        references_ds = proc_ds.map(lambda x: self.normalize_fn(x, text_field='ref', omit_columns=omit_columns))
         eval_data = {'pred': predictions_ds['rtext'], 'ref': references_ds['rtext']}
 
         # 4. compute amsr for given metric