temp update to exact_match

nl2bash_m.py

@@ -91,8 +91,8 @@ class nl2bash_m(evaluate.Metric):
         cmd_weight = 0.65,
         opt_weight = 0.25,
         arg_weight = 0.15,
-        ignore_case=False,
-        ignore_numbers=False,
+        ignore_case=True,
+        ignore_numbers=True,
     ):
 
         predictions = np.asarray(predictions)
@@ -110,37 +110,42 @@ class nl2bash_m(evaluate.Metric):
 
         final_score = 0
         for pred, refs in zip(predictions, references):
+            best_score = 0
+            for ref in refs:
+                if pred == ref:
+                    best_score = 1
+            final_score += best_score
 
-            if len(pred) == 0 and min([len(ref) for ref in refs]) == 0:
-                score = 1
-            elif len(pred) == 0 or min([len(ref) for ref in refs]) == 0:
-                score = 0
-            else:
-                best_score = 0
-                for ref in refs:
-                    pred_words, ref_words = pred.split(), ref.split()
+            # if len(pred) == 0 and min([len(ref) for ref in refs]) == 0:
+            #     score = 1
+            # elif len(pred) == 0 or min([len(ref) for ref in refs]) == 0:
+            #     score = 0
+            # else:
+            #     best_score = 0
+            #     for ref in refs:
+            #         pred_words, ref_words = pred.split(), ref.split()
 
                     
-                    # Get the cmd of predicted and ref 
-                    cmd_corr = 1 if pred_words.pop(0)==ref_words.pop(0) else 0
+            #         # Get the cmd of predicted and ref 
+            #         cmd_corr = 1 if pred_words.pop(0)==ref_words.pop(0) else 0
 
-                    # Get the option of predicted and ref
-                    pred_option = [ x for x in pred_words if x[0] == '-']
-                    ref_option = [ x for x in ref_words if x[0] == '-']
+            #         # Get the option of predicted and ref
+            #         pred_option = [ x for x in pred_words if x[0] == '-']
+            #         ref_option = [ x for x in ref_words if x[0] == '-']
                     
-                    # Get the arguments of predicted and ref
-                    pred_args = [ x for x in pred_words if x[0] != '-']
-                    ref_args = [ x for x in ref_words if x[0] != '-']
+            #         # Get the arguments of predicted and ref
+            #         pred_args = [ x for x in pred_words if x[0] != '-']
+            #         ref_args = [ x for x in ref_words if x[0] != '-']
 
-                    # Calculate scores
-                    cmd_score = cmd_weight * cmd_corr
-                    opt_score = opt_weight * self.get_score(pred_option, ref_option)
-                    arg_score = arg_weight * self.get_score(pred_args, ref_args)
+            #         # Calculate scores
+            #         cmd_score = cmd_weight * cmd_corr
+            #         opt_score = opt_weight * self.get_score(pred_option, ref_option)
+            #         arg_score = arg_weight * self.get_score(pred_args, ref_args)
 
-                    score = cmd_score + opt_score + arg_score 
-                    best_score = max(best_score, score)
+            #         score = cmd_score + opt_score + arg_score 
+            #         best_score = max(best_score, score)
 
-            final_score += best_score
+            # final_score += best_score
 
         final_score = final_score/len(predictions)