add new metric timestamp_delta

logmetric.py

@@ -188,10 +188,12 @@ class LogMetric(evaluate.Metric):
     def getTimestampsScore(self, pred_timestamps, ref_timestamps):
         timestamp_amt_score = self.smapeScore(len(pred_timestamps), len(ref_timestamps))
         
-        # if there are no predicted timestamps, return early. It is still consistent and monotonic.
-        if (len(pred_timestamps) == 0):
-            return timestamp_amt_score, 1.0, 1.0
+        if (len(pred_timestamps) == 0) and (len(ref_timestamps) == 0):
+            return timestamp_amt_score, 1.0, 1.0, 1.0
 
+        # if there are no predicted timestamps, return early. It is still consistent and monotonic.
+        if (len(pred_timestamps) == 0) and (len(ref_timestamps) != 0):
+            return timestamp_amt_score, 1.0, 1.0, 0.0
 
         # replace all digits in the reference timestamp (first timestamp) with '/d' to get
         # a regex that describes the format  
@@ -199,6 +201,7 @@ class LogMetric(evaluate.Metric):
         
         matchesPatternScore = 1.0
         monotonicallyIncreasingScore = 1.0
+        pred_timedeltas = []
         
         # A variable to save the previous timestamp (as datetime obj) to check monotonicity
         prev_datetime = None
@@ -212,7 +215,12 @@ class LogMetric(evaluate.Metric):
                 matchesPattern = re.fullmatch(pred_timestring_pattern, ts) is not None
                 # Check if the timestamps are monotonically increasing
                 cur_datetime = dateutil.parser.parse(ts)
-                monotonicallyIncreasing = True if prev_datetime == None else prev_datetime <= cur_datetime
+                if prev_datetime == None:
+                    monotonicallyIncreasing = True
+                else:
+                    monotonicallyIncreasing = prev_datetime <= cur_datetime
+                    pred_timedeltas.append((cur_datetime - prev_datetime).total_seconds())
+                
                 prev_datetime = cur_datetime
 
                 # If one entry doesn't fulfill the matching pattern property or the monotinicity property, set to 0 for whole log
@@ -224,9 +232,43 @@ class LogMetric(evaluate.Metric):
                 # e.g. date format not parsable by dateutil.parser
                 matchesPatternScore = 0.0
                 monotonicallyIncreasingScore = 0.0
+                pred_timedeltas.append(-1)
+
+
+            if (len(pred_timestamps) != 0) and (len(ref_timestamps) == 0):
+                return timestamp_amt_score, matchesPatternScore, monotonicallyIncreasingScore, 0.0
+
+
+        ref_timedeltas = []
+        prev_datetime = None
+        for i in range(len(ref_timestamps)):
+            ts = ref_timestamps[i]
+            try:
+                cur_datetime = dateutil.parser.parse(ts)
+                if prev_datetime == None:
+                    pass
+                else:
+                    ref_timedeltas.append((cur_datetime - prev_datetime).total_seconds())
+                
+                prev_datetime = cur_datetime
+
+            except Exception as e:
+               ref_timedeltas.append(-1)
+
+        minlength = min(len(pred_timedeltas), len(ref_timedeltas))
+
+        pred_timedeltas = pred_timedeltas[:minlength]
+        ref_timedeltas = ref_timedeltas[:minlength]
+
+        print("pred_timedeltas:", pred_timedeltas)
+        print("ref_timedeltas:", ref_timedeltas)
+
+
+        timestampDeltaScore = self.smapeScore(pred_timedeltas, ref_timedeltas)
         
+        print("timestampDeltaScore:", timestampDeltaScore)
         # matchesPatternScore and monotonicallyIncreasingScore are in {0,1}
-        return timestamp_amt_score, matchesPatternScore, monotonicallyIncreasingScore
+        return timestamp_amt_score, matchesPatternScore, monotonicallyIncreasingScore, timestampDeltaScore
 
 
 
@@ -270,11 +312,8 @@ class LogMetric(evaluate.Metric):
 
         linecontent_sacrebleu, linecontent_sacrebleu_withoutExplicitNumbers, linecontentlength_difference_SMAPE = self.getLineContentScore(pred_logMessages, ref_logMessages)
         
-        timestamps_difference_SMAPE, timestamps_formatConsistency_absolute, timestamps_monotinicity_absolute = self.getTimestampsScore(pred_timestamps, ref_timestamps)
+        timestamps_difference_SMAPE, timestamps_formatConsistency_absolute, timestamps_monotinicity_absolute, timestamps_delta_SMAPE = self.getTimestampsScore(pred_timestamps, ref_timestamps)
 
-        #TODO:
-        # linecontentordering_permutations = getLineContentOrderingScore(pred_logMessages, ref_logMessages)
-        
 
         # return weighted overall score of all the different scores
         return {"linecount_difference_SMAPE_score": linecount_difference_SMAPE,
@@ -283,7 +322,8 @@ class LogMetric(evaluate.Metric):
                 "linecontent_sacrebleu_withoutExplicitNumbers_score": linecontent_sacrebleu_withoutExplicitNumbers,
                 "timestamps_SMAPE_difference_score": timestamps_difference_SMAPE,
                 "timestamps_formatConsistency_score": timestamps_formatConsistency_absolute,
-                "timestamps_monotinicity_score": timestamps_monotinicity_absolute
+                "timestamps_monotinicity_score": timestamps_monotinicity_absolute,
+                "timestamps_delta_SMAPE_score" : timestamps_delta_SMAPE
                 }
 
     def _compute(self, predictions, references):