add a sacrebleu on the cleaned log-messages (ints and floats replaced by tokens)

logmetric.py

@@ -71,6 +71,13 @@ class LogMetric(evaluate.Metric):
     # Constant regex to get timestrings
     timestamp_regex = r'^\s*\[?\s*(\d{4}[-/.]\d{2}[-/.]\d{2}(?:[ T]\d{2}[:]\d{2}(?:[:]\d{2}(?:[.,]\d+)?)?(?:Z|[+-]\d{2}[:]\d{2})?)?)\s*\]?\s*'
     timestamp_pattern = re.compile(timestamp_regex, re.MULTILINE)
+
+    int_regex = r'(\s|^)(-?\d+)(\s|$)'
+    int_pattern = re.compile(int_regex)
+
+    float_regex = r'(\s|^)(-?\d+\.\d+)(\s|$)'
+    float_pattern = re.compile(float_regex)
+
     sacrebleu_metric = evaluate.load("evaluate-metric/sacrebleu")
 
 
@@ -153,6 +160,10 @@ class LogMetric(evaluate.Metric):
 
         return self.smapeScore(pred_lines_amt, ref_lines_amt)
     
+    def replaceNumbers(self, text:str):
+        text = self.int_pattern.sub(r'\1<|INT|>\3', text)
+        text = self.float_pattern.sub(r'\1<|FLOAT|\3>', text)
+        return text
     
     # Get differenct scores regarding the content of a log-message
     def getLineContentScore(self, pred_logMessages, ref_logMessages):
@@ -163,11 +174,17 @@ class LogMetric(evaluate.Metric):
         
         smape_length_score = self.get_length_score(pred_logMessages, ref_logMessages)
 
-        # Split the single log-messages (for jaccard)
-        preds_split = [p.split() for p in pred_logMessages]
-        refs_split = [r.split() for r in ref_logMessages]
+        vectorized_replaceNumbers = np.vectorize(self.replaceNumbers)
+
+        cleaned_pred_logMessages = vectorized_replaceNumbers(pred_logMessages)
+        cleaned_ref_logMessages = vectorized_replaceNumbers(ref_logMessages)
+        print("before:", pred_logMessages, "\n")
+        print("cleaned:", cleaned_pred_logMessages)
+
+        sacrebleu_withoutExplicitNumbers_score = self.sacrebleu_metric.compute(predictions=cleaned_pred_logMessages, references=cleaned_ref_logMessages)["score"] / 100.0
+
 
-        return sacrebleu_score, smape_length_score
+        return sacrebleu_score, sacrebleu_withoutExplicitNumbers_score, smape_length_score
     
     # get different scores regarding the timestamp
     def getTimestampsScore(self, pred_timestamps, ref_timestamps):
@@ -252,7 +269,7 @@ class LogMetric(evaluate.Metric):
         pred_logMessages += (max_logentries - len(pred_logMessages)) * [" "]
         ref_logMessages += (max_logentries- len(ref_logMessages)) * [" "]
 
-        linecontent_sacrebleu, linecontentlength_difference_SMAPE = self.getLineContentScore(pred_logMessages, ref_logMessages)
+        linecontent_sacrebleu, linecontent_sacrebleu_withoutExplicitNumbers, linecontentlength_difference_SMAPE = self.getLineContentScore(pred_logMessages, ref_logMessages)
         
         timestamps_difference_SMAPE, timestamps_formatConsistency_absolute, timestamps_monotinicity_absolute = self.getTimestampsScore(pred_timestamps, ref_timestamps)
 
@@ -264,6 +281,7 @@ class LogMetric(evaluate.Metric):
         return {"linecount_difference_SMAPE_score": linecount_difference_SMAPE,
                 "linecontentlength_difference_SMAPE_score": linecontentlength_difference_SMAPE,
                 "linecontent_sacrebleu_score": linecontent_sacrebleu,
+                "linecontent_sacrebleu_withoutExplicitNumbers_score": linecontent_sacrebleu_withoutExplicitNumbers,
                 "timestamps_SMAPE_difference_score": timestamps_difference_SMAPE,
                 "timestamps_formatConsistency_score": timestamps_formatConsistency_absolute,
                 "timestamps_monotinicity_score": timestamps_monotinicity_absolute